From 5621e59e42b70644b7692ae59c32f28125a88cd6 Mon Sep 17 00:00:00 2001 From: celogeek Date: Fri, 4 Mar 2022 16:59:26 +0100 Subject: [PATCH] store only chunk, record file --- internal/photos/api/db.go | 1 + internal/photos/api/errors.go | 9 +- internal/photos/api/file.go | 85 ++++++++++++------- internal/photos/api/main.go | 6 +- internal/photos/models/file.go | 1 + internal/photos/models/filechunk.go | 12 +++ internal/photoserrors/store.go | 16 ++-- internal/store/core.go | 124 ++++++++++------------------ 8 files changed, 133 insertions(+), 121 deletions(-) create mode 100644 internal/photos/models/filechunk.go diff --git a/internal/photos/api/db.go b/internal/photos/api/db.go index 83548ea..1e10cef 100644 --- a/internal/photos/api/db.go +++ b/internal/photos/api/db.go @@ -16,6 +16,7 @@ func (s *Service) Migrate() { tx.AutoMigrate(&models.Account{}) tx.AutoMigrate(&models.Session{}) tx.AutoMigrate(&models.File{}) + tx.AutoMigrate(&models.FileChunk{}) } func (s *Service) DBConfig() { diff --git a/internal/photos/api/errors.go b/internal/photos/api/errors.go index eddc7f1..92bc1f5 100644 --- a/internal/photos/api/errors.go +++ b/internal/photos/api/errors.go @@ -5,8 +5,15 @@ import ( ) func (s *Service) Error(c *gin.Context, code int, err error) { + var status string + if code >= 200 && code < 400 { + status = "success" + } else { + status = "failed" + } + c.AbortWithStatusJSON(code, gin.H{ - "status": "failed", + "status": status, "error": err.Error(), }) } diff --git a/internal/photos/api/file.go b/internal/photos/api/file.go index 302523b..56be7dc 100644 --- a/internal/photos/api/file.go +++ b/internal/photos/api/file.go @@ -2,20 +2,23 @@ package api import ( "bytes" - "fmt" + "errors" "io" "net/http" - "path/filepath" + "strings" "github.com/gin-gonic/gin" + "gitlab.celogeek.com/photos/api/internal/photos/models" "gitlab.celogeek.com/photos/api/internal/photoserrors" + "gorm.io/gorm" ) var CHUNK_SIZE int64 = 1 << 20 type File struct { - Sum string `json:"sum"` - Chunks []string `json:"chunks"` + Name string `json:"name"` + Checksum string `json:"checksum"` + Chunks []string `json:"chunks"` } func (s *Service) FileCreate(c *gin.Context) { @@ -25,7 +28,12 @@ func (s *Service) FileCreate(c *gin.Context) { return } - if len(file.Sum) != 40 { + if len(file.Name) < 1 { + s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreMissingName) + return + } + + if len(file.Checksum) != 40 { s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreBadChecksum) return } @@ -41,48 +49,61 @@ func (s *Service) FileCreate(c *gin.Context) { } } - r, rs, err := s.Store.CombineTemp(file.Sum, file.Chunks) + r, rs, err := s.Store.Combine(file.Chunks) if err != nil { - s.Error(c, http.StatusInternalServerError, err) + if strings.HasPrefix(err.Error(), "chunk") && strings.HasSuffix(err.Error(), "doesn't exists") { + s.Error(c, http.StatusBadRequest, err) + } else { + s.Error(c, http.StatusInternalServerError, err) + } return } - if r != file.Sum { - fmt.Printf("R=%s, O=%s\n", r, file.Sum) + if r != file.Checksum { s.Error(c, http.StatusExpectationFailed, photoserrors.ErrStoreMismatchChecksum) return } - if err = s.Store.CommitTemp(file.Sum, file.Chunks); err != nil { + sess := s.CurrentSession(c) + + f := &models.File{ + Name: file.Name, + Checksum: file.Checksum, + Size: rs, + AuthorId: &sess.AccountId, + } + + err = s.DB.Transaction(func(tx *gorm.DB) error { + if err := tx.Create(f).Error; err != nil { + return err + } + for i, chunk := range file.Chunks { + fc := &models.FileChunk{ + FileId: f.ID, + Part: uint32(i + 1), + Checksum: chunk, + } + if err := tx.Create(fc).Error; err != nil { + return err + } + } + return nil + }) + + if err != nil { s.Error(c, http.StatusInternalServerError, err) return } c.JSON(http.StatusOK, gin.H{ "status": "success", - "sum": file.Sum, + "sum": file.Checksum, "nbChunks": len(file.Chunks), "size": rs, }) } -func (s *Service) FileCreateTemp(c *gin.Context) { - var ( - origsum = c.Param("origsum") - sumb = c.Param("sum") - ) - if len(origsum) != 40 || len(sumb) != 40 { - s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreBadChecksum) - return - } - - originaldir := s.Store.Dir("original", origsum) - originalname := filepath.Join(originaldir, origsum) - if s.Store.FileExists(originalname) { - s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreAlreadyExists) - return - } - +func (s *Service) FileCreateChunk(c *gin.Context) { if c.Request.ContentLength > CHUNK_SIZE { s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreBadChunkSize) return @@ -92,8 +113,12 @@ func (s *Service) FileCreateTemp(c *gin.Context) { io.Copy(b, c.Request.Body) c.Request.Body.Close() - if err := s.Store.SaveTemp(origsum, sumb, b.Bytes()); err != nil { - s.Error(c, http.StatusBadRequest, err) + if err := s.Store.NewChunk(b.Bytes()).Save(); err != nil { + if errors.Is(err, photoserrors.ErrStoreChunkAlreadyExists) { + s.Error(c, http.StatusOK, err) + } else { + s.Error(c, http.StatusBadRequest, err) + } return } diff --git a/internal/photos/api/main.go b/internal/photos/api/main.go index 07f4d8e..21f2027 100644 --- a/internal/photos/api/main.go +++ b/internal/photos/api/main.go @@ -60,7 +60,7 @@ func (s *Service) SetupRoutes() { album := s.Gin.Group("/file") album.Use(s.RequireSession) album.POST("/", s.FileCreate) - album.POST("/tmp/:origsum/:sum", s.FileCreateTemp) + album.POST("/chunk", s.FileCreateChunk) s.Gin.NoRoute(func(c *gin.Context) { s.Error(c, http.StatusNotFound, photoserrors.ErrReqNotFound) @@ -75,10 +75,8 @@ func (s *Service) PrepareStore() { if !d.IsDir() { s.LogErr.Fatal("Store", photoserrors.ErrStorePathNotADirectory) } - if err := s.Store.MkDirs([]string{"tmp", "original"}); err != nil { - s.LogErr.Fatal("Store", err) - } } + func (s *Service) Run() error { rand.Seed(time.Now().UnixNano()) s.PrepareStore() diff --git a/internal/photos/models/file.go b/internal/photos/models/file.go index 4c8c217..0219bd4 100644 --- a/internal/photos/models/file.go +++ b/internal/photos/models/file.go @@ -6,6 +6,7 @@ type File struct { ID uint32 `gorm:"primary_key" json:"id"` Name string `gorm:"not null" json:"name"` Checksum string `gorm:"unique;size:44;not null"` + Size uint64 `gorm:"not null"` Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"` AuthorId *uint32 `json:"-"` CreatedAt time.Time `json:"created_at"` diff --git a/internal/photos/models/filechunk.go b/internal/photos/models/filechunk.go new file mode 100644 index 0000000..92009a9 --- /dev/null +++ b/internal/photos/models/filechunk.go @@ -0,0 +1,12 @@ +package models + +import "time" + +type FileChunk struct { + FileId uint32 + File *File `gorm:"constraint:OnDelete:CASCADE,OnUpdate:CASCADE"` + Part uint32 + Checksum string `gorm:"unique;size:44;not null"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/internal/photoserrors/store.go b/internal/photoserrors/store.go index 36dfe93..22ec2a1 100644 --- a/internal/photoserrors/store.go +++ b/internal/photoserrors/store.go @@ -4,11 +4,13 @@ import "errors" var ( // Store - ErrStorePathNotADirectory = errors.New("store path is not a directory") - ErrStoreBadChecksum = errors.New("checksum should be sha1 in hex format") - ErrStoreBadChunkSize = errors.New("part file size should be 1MB max") - ErrStoreMissingChunks = errors.New("part checksum missing") - ErrStoreWrongChecksum = errors.New("wrong checksum") - ErrStoreMismatchChecksum = errors.New("part files doesn't match the original checksum") - ErrStoreAlreadyExists = errors.New("original file already exists") + ErrStorePathNotADirectory = errors.New("store path is not a directory") + ErrStoreBadChecksum = errors.New("checksum should be sha1 in hex format") + ErrStoreBadChunkSize = errors.New("part file size should be 1MB max") + ErrStoreMissingChunks = errors.New("part checksum missing") + ErrStoreWrongChecksum = errors.New("wrong checksum") + ErrStoreMismatchChecksum = errors.New("part files doesn't match the original checksum") + ErrStoreAlreadyExists = errors.New("original file already exists") + ErrStoreChunkAlreadyExists = errors.New("chunk file already exists") + ErrStoreMissingName = errors.New("name required") ) diff --git a/internal/store/core.go b/internal/store/core.go index fd2c184..5b3d6d5 100644 --- a/internal/store/core.go +++ b/internal/store/core.go @@ -15,115 +15,81 @@ type Store struct { Path string } -func (s *Store) Dir(path, sum string) string { - return filepath.Join(s.Path, path, sum[0:1], sum[1:2]) +type Chunk struct { + *Store + Sum string + Bytes []byte } -func (s *Store) MkDirs(dirs []string) error { - for _, dir := range dirs { - if err := os.MkdirAll(filepath.Join(s.Path, dir), 0755); err != nil { - return err - } +func (s *Store) NewChunk(b []byte) *Chunk { + sum := sha1.New() + sum.Write(b) + sumString := hex.EncodeToString(sum.Sum(nil)) + + return &Chunk{s, sumString, b} +} + +func (s *Store) LoadChunk(sum string) (*Chunk, error) { + c := &Chunk{s, sum, nil} + if !c.FileExists() { + return nil, fmt.Errorf("chunk %s doesn't exists", sum) } - return nil + b, err := os.ReadFile(c.Filename()) + if err != nil { + return nil, err + } + c.Bytes = b + return c, nil } -func (s *Store) FileExists(filename string) bool { - fs, err := os.Stat(filename) +func (c *Chunk) Dir() string { + return filepath.Join(c.Path, "storage", c.Sum[0:1], c.Sum[1:2]) +} + +func (c *Chunk) Filename() string { + return filepath.Join(c.Dir(), c.Sum) +} + +func (c *Chunk) FileExists() bool { + fs, err := os.Stat(c.Filename()) if errors.Is(err, os.ErrNotExist) { return false } return !fs.IsDir() } -func (s *Store) SaveTemp(path string, sumb string, b []byte) error { - sum := sha1.New() - sum.Write(b) - sumString := hex.EncodeToString(sum.Sum(nil)) +func (c *Chunk) Mkdir() error { + return os.MkdirAll(c.Dir(), 0755) +} - if sumb != sumString { - return photoserrors.ErrStoreWrongChecksum +func (c *Chunk) Save() error { + if c.FileExists() { + return photoserrors.ErrStoreChunkAlreadyExists } - tmpdir := filepath.Join("tmp", path) - dir := s.Dir(tmpdir, sumString) - filename := filepath.Join(dir, sumString) - - if s.FileExists(filename) { - return nil - } - - if err := os.MkdirAll(dir, 0755); err != nil { + if err := c.Mkdir(); err != nil { return err } - fs, err := os.Create(filename) + fs, err := os.Create(c.Filename()) if err != nil { return err } defer fs.Close() - _, err = fs.Write(b) + _, err = fs.Write(c.Bytes) return err } -func (s *Store) CombineTemp(path string, sumb []string) (string, uint64, error) { - tmpdir := filepath.Join("tmp", path) +func (s *Store) Combine(sumb []string) (string, uint64, error) { sum := sha1.New() size := uint64(0) for _, sb := range sumb { - dir := s.Dir(tmpdir, sb) - filename := filepath.Join(dir, sb) - if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { - return "", 0, fmt.Errorf("%s: chunk %s doesn't exists", path, sb) - } - b, err := os.ReadFile(filename) + c, err := s.LoadChunk(sb) if err != nil { return "", 0, err } - sum.Write(b) - size += uint64(len(b)) + sum.Write(c.Bytes) + size += uint64(len(c.Bytes)) } return hex.EncodeToString(sum.Sum(nil)), size, nil } - -func (s *Store) CommitTemp(path string, sumb []string) error { - tmpdir := filepath.Join("tmp", path) - originaldir := s.Dir("original", path) - originalname := filepath.Join(originaldir, path) - - if s.FileExists(originalname) { - return photoserrors.ErrStoreAlreadyExists - } - - os.MkdirAll(originaldir, 0755) - - fs, err := os.Create(originalname) - if err != nil { - return err - } - defer fs.Close() - - for _, sb := range sumb { - dir := s.Dir(tmpdir, sb) - filename := filepath.Join(dir, sb) - if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("%s: chunk %s doesn't exists", path, sb) - } - b, err := os.ReadFile(filename) - if err != nil { - return err - } - if _, err := fs.Write(b); err != nil { - return err - } - } - - for _, sb := range sumb { - dir := s.Dir(tmpdir, sb) - filename := filepath.Join(dir, sb) - if err := os.Remove(filename); err != nil { - return err - } - } - return os.RemoveAll(filepath.Join(s.Path, tmpdir)) -}