From 7557e748622f45bc2a54c8a19abaee075aa11dbc Mon Sep 17 00:00:00 2001 From: celogeek Date: Tue, 1 Mar 2022 18:20:54 +0100 Subject: [PATCH] use chunks list and store --- cmd/photos-api/main.go | 4 +- internal/photos/api/errors.go | 3 +- internal/photos/api/file.go | 168 ++++++++-------------------------- internal/photos/api/main.go | 19 +++- internal/store/core.go | 118 ++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 134 deletions(-) create mode 100644 internal/store/core.go diff --git a/cmd/photos-api/main.go b/cmd/photos-api/main.go index 06b6f2e..a63d226 100644 --- a/cmd/photos-api/main.go +++ b/cmd/photos-api/main.go @@ -23,7 +23,9 @@ var ( ) func main() { - config := &api.ServiceConfig{DB: mysql.NewConfig()} + config := &api.ServiceConfig{ + DB: mysql.NewConfig(), + } flag.StringVar(&config.Listen, "listen", listen, "Listen address") flag.StringVar(&config.DB.Addr, "mysql-addr", mysqlAddr, "Mysql addr") diff --git a/internal/photos/api/errors.go b/internal/photos/api/errors.go index cde4acc..52c6b6c 100644 --- a/internal/photos/api/errors.go +++ b/internal/photos/api/errors.go @@ -30,8 +30,7 @@ var ( ErrStorePathNotADirectory = errors.New("store path is not a directory") ErrStoreBadChecksum = errors.New("checksum should be sha1 in hex format") ErrStoreBadChunkSize = errors.New("part file size should be 1MB max") - ErrStoreWrongPartChecksum = errors.New("part file wrong checksum") - ErrStoreBadPartNumber = errors.New("part should be a positive number") + ErrStoreMissingChunks = errors.New("part checksum missing") ErrStoreMismatchChecksum = errors.New("part files doesn't match the original checksum") ) diff --git a/internal/photos/api/file.go b/internal/photos/api/file.go index 2a718dd..84cad5d 100644 --- a/internal/photos/api/file.go +++ b/internal/photos/api/file.go @@ -2,140 +2,74 @@ package api import ( "bytes" - "crypto/sha1" - "encoding/hex" "fmt" "io" - "io/fs" "net/http" - "os" - "path/filepath" - "sort" - "strconv" "github.com/gin-gonic/gin" ) var CHUNK_SIZE int64 = 1 << 20 -func (s *Service) PrepareStore() { - d, err := os.Stat(s.Config.StorePath) - if err != nil { - s.LogErr.Fatal("Store", err) - } - if !d.IsDir() { - s.LogErr.Fatal("Store", ErrStorePathNotADirectory) - } -} - -func (s *Service) StoreDir(checksum string) (string, error) { - dir := filepath.Join(s.Config.StorePath, "original", checksum[0:1], checksum[1:2], checksum) - err := os.MkdirAll(dir, 0755) - return dir, err -} - -func (s *Service) TempDir(checksum string) (string, error) { - dir := filepath.Join(s.Config.StorePath, "tmp", checksum) - err := os.MkdirAll(dir, 0755) - return dir, err -} - -type FileChunks struct { - N uint64 - Path fs.FS - Name string -} - -func (s *Service) FileChunks(checksum string) ([]FileChunks, error) { - base := filepath.Join(s.Config.StorePath, "tmp", checksum) - baseDir := os.DirFS(base) - dir, err := os.Open(base) - if err != nil { - return nil, err - } - defer dir.Close() - - files, err := dir.Readdirnames(-1) - if err != nil { - return nil, err - } - parts := []FileChunks{} - for _, f := range files { - n, err := strconv.ParseUint(f, 10, 64) - if err != nil { - continue - } - parts = append(parts, FileChunks{n, baseDir, f}) - } - - sort.Slice(parts, func(i, j int) bool { - return parts[i].N < parts[j].N - }) - return parts, nil +type File struct { + Sum string `json:"sum"` + Chunks []string `json:"chunks"` } func (s *Service) FileCreate(c *gin.Context) { - var originalChecksum = c.Param("original_checksum") - files, err := s.FileChunks(originalChecksum) - if err != nil { + file := &File{} + if err := c.ShouldBindJSON(file); err != nil { s.Error(c, http.StatusInternalServerError, err) return } - dir, err := s.StoreDir(originalChecksum) + + if len(file.Sum) != 40 { + s.Error(c, http.StatusBadRequest, ErrStoreBadChecksum) + return + } + if len(file.Chunks) == 0 { + s.Error(c, http.StatusBadRequest, ErrStoreMissingChunks) + return + } + + for _, chunk := range file.Chunks { + if len(chunk) != 40 { + s.Error(c, http.StatusBadRequest, ErrStoreBadChecksum) + return + } + } + + r, rs, err := s.Store.CombineTemp(file.Sum, file.Chunks) if err != nil { s.Error(c, http.StatusInternalServerError, err) return } - data, err := os.Create(filepath.Join(dir, ".tmp")) - if err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - defer data.Close() - - sum := sha1.New() - size := uint64(0) - for _, f := range files { - b, err := fs.ReadFile(f.Path, f.Name) - if err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - _, err = sum.Write(b) - if err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - _, err = data.Write(b) - if err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - size += uint64(len(b)) - } - r := hex.EncodeToString(sum.Sum(nil)) - if r != originalChecksum { - fmt.Printf("R=%s, O=%s\n", r, originalChecksum) + if r != file.Sum { + fmt.Printf("R=%s, O=%s\n", r, file.Sum) s.Error(c, http.StatusExpectationFailed, ErrStoreMismatchChecksum) return } + if err = s.Store.CommitTemp(file.Sum, file.Chunks); err != nil { + s.Error(c, http.StatusInternalServerError, err) + return + } + c.JSON(http.StatusOK, gin.H{ "status": "success", - "checksum": originalChecksum, - "nbParts": len(files), - "size": size, + "sum": file.Sum, + "nbChunks": len(file.Chunks), + "size": rs, }) } -func (s *Service) FileChunk(c *gin.Context) { +func (s *Service) FileCreateTemp(c *gin.Context) { var ( - originalChecksum = c.Param("original_checksum") - part = c.Param("part") - partChecksum = c.Param("part_checksum") + origsum = c.Param("origsum") + sumb = c.Param("sum") ) - if len(originalChecksum) != 40 || len(partChecksum) != 40 { + if len(origsum) != 40 || len(sumb) != 40 { s.Error(c, http.StatusBadRequest, ErrStoreBadChecksum) return } @@ -145,39 +79,15 @@ func (s *Service) FileChunk(c *gin.Context) { return } - p, err := strconv.ParseUint(part, 10, 64) - if err != nil || p < 1 { - s.Error(c, http.StatusBadRequest, ErrStoreBadPartNumber) - return - } - b := bytes.NewBuffer([]byte{}) io.Copy(b, c.Request.Body) c.Request.Body.Close() - sum := sha1.New() - sum.Write(b.Bytes()) - r := hex.EncodeToString(sum.Sum(nil)) - - if partChecksum != r { - s.Error(c, http.StatusBadRequest, ErrStoreWrongPartChecksum) + if err := s.Store.SaveTemp(origsum, sumb, b.Bytes()); err != nil { + s.Error(c, http.StatusBadRequest, err) return } - dir, err := s.TempDir(originalChecksum) - if err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - - f, err := os.Create(filepath.Join(dir, part)) - if err != nil { - s.Error(c, http.StatusInternalServerError, err) - return - } - f.Write(b.Bytes()) - f.Close() - c.JSON(http.StatusOK, gin.H{ "status": "success", }) diff --git a/internal/photos/api/main.go b/internal/photos/api/main.go index 327f937..053cd51 100644 --- a/internal/photos/api/main.go +++ b/internal/photos/api/main.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" "github.com/go-sql-driver/mysql" + "gitlab.celogeek.com/photos/api/internal/store" "gorm.io/gorm" ) @@ -17,6 +18,7 @@ type Service struct { Gin *gin.Engine DB *gorm.DB Config *ServiceConfig + Store *store.Store LogOk *Logger LogErr *Logger } @@ -31,6 +33,7 @@ func New(config *ServiceConfig) *Service { return &Service{ Gin: gin.New(), Config: config, + Store: &store.Store{Path: config.StorePath}, LogOk: &Logger{os.Stdout, "Photos"}, LogErr: &Logger{os.Stderr, "Photos"}, } @@ -55,14 +58,26 @@ func (s *Service) SetupRoutes() { album := s.Gin.Group("/file") album.Use(s.RequireSession) - album.POST("/:original_checksum", s.FileCreate) - album.POST("/:original_checksum/:part/:part_checksum", s.FileChunk) + album.POST("/", s.FileCreate) + album.POST("/tmp/:origsum/:sum", s.FileCreateTemp) s.Gin.NoRoute(func(c *gin.Context) { s.Error(c, http.StatusNotFound, ErrReqNotFound) }) } +func (s *Service) PrepareStore() { + d, err := os.Stat(s.Store.Path) + if err != nil { + s.LogErr.Fatal("Store", err) + } + if !d.IsDir() { + s.LogErr.Fatal("Store", ErrStorePathNotADirectory) + } + if err := s.Store.MkDirs([]string{"tmp", "original"}); err != nil { + s.LogErr.Fatal("Store", err) + } +} func (s *Service) Run() error { rand.Seed(time.Now().UnixNano()) s.PrepareStore() diff --git a/internal/store/core.go b/internal/store/core.go new file mode 100644 index 0000000..e41d5df --- /dev/null +++ b/internal/store/core.go @@ -0,0 +1,118 @@ +package store + +import ( + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" +) + +type Store struct { + Path string +} + +func (s *Store) Dir(path, sum string) string { + return filepath.Join(s.Path, path, sum[0:1], sum[1:2]) +} + +func (s *Store) MkDirs(dirs []string) error { + for _, dir := range dirs { + if err := os.MkdirAll(filepath.Join(s.Path, dir), 0755); err != nil { + return err + } + } + return nil +} + +func (s *Store) FileExists(filename string) bool { + fs, err := os.Stat(filename) + if errors.Is(err, os.ErrNotExist) { + return false + } + return !fs.IsDir() +} + +func (s *Store) SaveTemp(path string, sumb string, b []byte) error { + sum := sha1.New() + sum.Write(b) + sumString := hex.EncodeToString(sum.Sum(nil)) + + if sumb != sumString { + return errors.New("wrong checksum") + } + + dir := s.Dir(path, sumString) + filename := filepath.Join(dir, sumString) + + if s.FileExists(filename) { + return nil + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + + fs, err := os.Create(filename) + if err != nil { + return err + } + defer fs.Close() + _, err = fs.Write(b) + return err +} + +func (s *Store) CombineTemp(path string, sumb []string) (string, uint64, error) { + tmpdir := filepath.Join("tmp", path) + sum := sha1.New() + size := uint64(0) + for _, sb := range sumb { + dir := s.Dir(tmpdir, sb) + filename := filepath.Join(dir, sb) + if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { + return "", 0, fmt.Errorf("%s: chunk %s doesn't exists", path, sb) + } + b, err := os.ReadFile(filename) + if err != nil { + return "", 0, err + } + sum.Write(b) + size += uint64(len(b)) + } + return hex.EncodeToString(sum.Sum(nil)), size, nil +} + +func (s *Store) CommitTemp(path string, sumb []string) error { + tmpdir := filepath.Join("tmp", path) + originaldir := s.Dir("original", path) + originalname := filepath.Join(originaldir, path) + + if s.FileExists(originalname) { + return fmt.Errorf("original file already exists") + } + + os.MkdirAll(originaldir, 0755) + + fs, err := os.Create(originalname) + if err != nil { + return err + } + defer fs.Close() + + for _, sb := range sumb { + dir := s.Dir(tmpdir, sb) + filename := filepath.Join(dir, sb) + if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s: chunk %s doesn't exists", path, sb) + } + b, err := os.ReadFile(filename) + if err != nil { + return err + } + if _, err := fs.Write(b); err != nil { + return err + } + } + return nil +}