package photosapi import ( "errors" "fmt" "io" "net/http" "os" "strings" "time" "github.com/gin-gonic/gin" "github.com/google/uuid" ) const ( MaxUploadPartSize = 4 << 20 // 4MB ) var ( ErrUploadNotExists = errors.New("upload id does not exists") ErrUploadPartTooLarge = fmt.Errorf("upload part too large (> %d B)", MaxUploadPartSize) ErrUploadPartWrongSha256 = errors.New("upload part sha256 does not match") ErrFileAlreadExists = errors.New("file already exists") ErrUploadPartsCombineWrongSha256 = errors.New("upload parts combined sha256 does not match") ) // Model type File struct { ID uint32 `gorm:"primary_key" json:"id"` Name string `gorm:"not null" json:"name"` Checksum string `gorm:"unique;size:44;not null"` Size uint64 `gorm:"not null"` Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"` AuthorId *uint32 `json:"-"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } func (s *Service) UploadCreate(c *gin.Context) { sha, err := uuid.NewRandom() if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } upload := &Upload{sha.String()} if err := s.StorageTmp.Create(upload.Id); err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } c.JSON(http.StatusCreated, upload) } // Service type Upload struct { Id string `json:"upload_id" uri:"upload_id" binding:"required,uuid"` } type UploadPartQuery struct { Part uint `form:"part" binding:"required"` PartSha256 string `form:"sha256" binding:"required,sha256"` } type UploadCompleteRequest struct { Sha256 string `json:"sha256" binding:"required,sha256"` Name string `json:"name" binding:"required"` Parts uint `json:"parts" binding:"required"` } type UploadCompleteFileOptions struct { Ext string IsTemp bool } func (u *UploadCompleteRequest) Paths() []string { return []string{u.Sha256[0:1], u.Sha256[1:2]} } func (u *UploadCompleteRequest) File(options *UploadCompleteFileOptions) []string { filename := []string{} if options == nil { options = &UploadCompleteFileOptions{} } if options.IsTemp { filename = append(filename, "._tmp_") } filename = append(filename, u.Sha256) if len(options.Ext) > 0 { filename = append(filename, ".", options.Ext) } return []string{ u.Sha256[0:1], u.Sha256[1:2], strings.Join(filename, ""), } } func (s *Service) UploadPart(c *gin.Context) { var ( upload Upload uploadPart UploadPartQuery ) if c.BindUri(&upload) != nil || c.BindQuery(&uploadPart) != nil { return } if !s.StorageTmp.Exists(upload.Id) { c.AbortWithError(http.StatusNotFound, ErrUploadNotExists) return } if c.Request.ContentLength > MaxUploadPartSize { c.AbortWithError(http.StatusRequestEntityTooLarge, ErrUploadPartTooLarge) return } tmp_file := s.StorageTmp.Join(upload.Id, fmt.Sprintf("._tmp_%d", uploadPart.Part)) file := s.StorageTmp.Join(upload.Id, fmt.Sprint(uploadPart.Part)) f, err := os.Create(tmp_file) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } sha := NewChecksum() t := io.TeeReader(c.Request.Body, sha) _, err = io.Copy(f, t) if err != nil { f.Close() os.Remove(tmp_file) c.AbortWithError(http.StatusInternalServerError, err) return } f.Close() if !sha.Match(uploadPart.PartSha256) { os.Remove(tmp_file) c.AbortWithError(http.StatusBadRequest, ErrUploadPartWrongSha256) return } err = os.Rename(tmp_file, file) if err != nil { os.Remove(tmp_file) c.AbortWithError(http.StatusInternalServerError, err) return } c.Status(http.StatusNoContent) } func (s *Service) UploadCancel(c *gin.Context) { var upload Upload if c.BindUri(&upload) != nil { return } if err := s.StorageTmp.Delete(upload.Id); err != nil { c.AbortWithError(http.StatusNotFound, ErrUploadNotExists) return } c.Status(http.StatusNoContent) } func (s *Service) UploadComplete(c *gin.Context) { var ( upload Upload uploadCompleteRequest UploadCompleteRequest ) if c.BindUri(&upload) != nil || c.BindJSON(&uploadCompleteRequest) != nil { return } if !s.StorageTmp.Exists(upload.Id) { c.AbortWithError(http.StatusNotFound, ErrUploadNotExists) return } if f, err := s.StorageUpload.Stat(uploadCompleteRequest.File(nil)...); err == nil && f.Mode().IsRegular() { c.AbortWithError(http.StatusConflict, ErrFileAlreadExists) return } if err := s.StorageUpload.Create(uploadCompleteRequest.Paths()...); err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } f, err := os.Create(s.StorageUpload.Join(uploadCompleteRequest.File(&UploadCompleteFileOptions{IsTemp: true})...)) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } fsha := NewChecksum() ft := io.MultiWriter(f, fsha) size := uint64(0) defer f.Close() for part := uint(1); part <= uploadCompleteRequest.Parts; part++ { p, err := os.Open(s.StorageTmp.Join(upload.Id, fmt.Sprint(part))) if err != nil { c.AbortWithError(http.StatusNotFound, fmt.Errorf("upload part %d missing", part)) return } w, err := io.Copy(ft, p) if err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } size += uint64(w) } if !fsha.Match(uploadCompleteRequest.Sha256) { c.AbortWithError(http.StatusExpectationFailed, ErrUploadPartsCombineWrongSha256) return } f.Close() if err := os.Rename( s.StorageUpload.Join(uploadCompleteRequest.File(&UploadCompleteFileOptions{IsTemp: true})...), s.StorageUpload.Join(uploadCompleteRequest.File(nil)...), ); err != nil { c.AbortWithError(http.StatusInternalServerError, err) return } sess := s.CurrentSession(c) record := &File{ Name: uploadCompleteRequest.Name, Checksum: uploadCompleteRequest.Sha256, AuthorId: &sess.Account.ID, Size: size, } if err := s.DB.Create(record).Error; err != nil { c.AbortWithError(http.StatusConflict, err) return } c.Status(http.StatusNoContent) } func (s *Service) UploadInit() { upload := s.Gin.Group("/upload") upload.Use(s.RequireSession) // start upload.POST("", s.UploadCreate) // Cancel upload.DELETE("/:upload_id", s.UploadCancel) // Add part upload.PUT("/:upload_id", s.UploadPart) // Complete upload.POST("/:upload_id", s.UploadComplete) }