diff --git a/cmd/photos-api-cli/upload.go b/cmd/photos-api-cli/upload.go index 9dea438..f7f7355 100644 --- a/cmd/photos-api-cli/upload.go +++ b/cmd/photos-api-cli/upload.go @@ -12,6 +12,7 @@ import ( "github.com/go-resty/resty/v2" "github.com/schollz/progressbar/v3" + "gitlab.celogeek.com/photos/api/internal/photos/api" ) type UploadCommand struct { @@ -52,11 +53,6 @@ func (c *UploadCommand) FileExists() (string, error) { if err != nil { return "", err } - chunkSize := int64(1 << 20) - nbChunks := st.Size() / chunkSize - if st.Size()%chunkSize > 0 { - nbChunks++ - } progress := progressbar.DefaultBytes(st.Size(), fmt.Sprintf("Checking %s", filepath.Base(c.File))) defer progress.Close() @@ -86,9 +82,8 @@ func (c *UploadCommand) FileUpload(sum string) error { if err != nil { return err } - chunkSize := int64(1 << 20) - nbChunks := st.Size() / chunkSize - if st.Size()%chunkSize > 0 { + nbChunks := st.Size() / api.CHUNK_SIZE + if st.Size()%api.CHUNK_SIZE > 0 { nbChunks++ } @@ -110,7 +105,7 @@ func (c *UploadCommand) FileUpload(sum string) error { for w := uint32(0); w < c.Workers; w++ { go func(w uint32) { defer wg.Done() - b := make([]byte, chunkSize) + b := make([]byte, api.CHUNK_SIZE) for { mu.Lock() part := i diff --git a/internal/photos/api/file.go b/internal/photos/api/file.go index c495ef6..65441f7 100644 --- a/internal/photos/api/file.go +++ b/internal/photos/api/file.go @@ -3,8 +3,11 @@ package api import ( "bytes" "errors" + "fmt" "io" + "mime" "net/http" + "path/filepath" "strings" "github.com/gin-gonic/gin" @@ -14,7 +17,7 @@ import ( "gorm.io/gorm" ) -var CHUNK_SIZE int64 = 1 << 20 +var CHUNK_SIZE int64 = 4 << 20 type File struct { Name string `json:"name"` @@ -151,7 +154,7 @@ func (s *Service) FileChunkExists(c *gin.Context) { return } - if s.Store.ChunkExists(checksum) { + if s.Store.Chunk(checksum).FileExists() { c.Status(http.StatusOK) } else { c.Status(http.StatusNotFound) @@ -177,3 +180,44 @@ func (s *Service) FileExists(c *gin.Context) { c.Status(http.StatusNotFound) } } + +func (s *Service) FileGet(c *gin.Context) { + checksum := c.Param("checksum") + if len(checksum) != 40 { + s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreBadChecksum) + return + } + if checksum == c.GetHeader("If-None-Match") { + c.Status(http.StatusNotModified) + return + } + + f := &models.File{} + if err := s.DB.Debug().Preload("Chunks").Where("checksum = ?", checksum).First(&f).Error; err != nil { + s.Error(c, http.StatusBadRequest, err) + return + } + + chunks := make([]string, len(f.Chunks)) + for _, fc := range f.Chunks { + chunks[fc.Part-1] = fc.Checksum + } + + reader, err := s.Store.NewStoreReader(chunks) + if err != nil { + s.Error(c, http.StatusInternalServerError, err) + return + } + defer reader.Close() + + c.DataFromReader( + http.StatusOK, + reader.Size, + mime.TypeByExtension(filepath.Ext(f.Name)), + reader, + map[string]string{ + "Content-Disposition": fmt.Sprintf("inline; filename=\"%s\"", f.Name), + "ETag": f.Checksum, + }, + ) +} diff --git a/internal/photos/api/main.go b/internal/photos/api/main.go index efaed89..00bfb44 100644 --- a/internal/photos/api/main.go +++ b/internal/photos/api/main.go @@ -61,6 +61,7 @@ func (s *Service) SetupRoutes() { album.Use(s.RequireSession) album.POST("", s.FileCreate) album.HEAD("/:checksum", s.FileExists) + album.GET("/:checksum", s.FileGet) album.POST("/chunk", s.FileCreateChunk) album.HEAD("/chunk/:checksum", s.FileChunkExists) diff --git a/internal/photos/api/session.go b/internal/photos/api/session.go index 802b707..2410a58 100644 --- a/internal/photos/api/session.go +++ b/internal/photos/api/session.go @@ -13,13 +13,20 @@ import ( ) func (s *Service) RequireAuthToken(c *gin.Context) { - token := c.GetHeader("Authorization") - if !strings.HasPrefix(token, "Private ") { + tokenAuth := c.GetHeader("Authorization") + tokenCookie, _ := c.Cookie("photoapitoken") + + if tokenAuth != "" { + if !strings.HasPrefix(tokenAuth, "Private ") { + s.Error(c, http.StatusForbidden, photoserrors.ErrTokenMissing) + } else { + c.Set("token", tokenAuth[8:]) + } + } else if tokenCookie != "" { + c.Set("token", tokenCookie) + } else { s.Error(c, http.StatusForbidden, photoserrors.ErrTokenMissing) - return } - token = token[8:] - c.Set("token", token) } func (s *Service) RequireSession(c *gin.Context) { diff --git a/internal/photos/models/file.go b/internal/photos/models/file.go index 0219bd4..f51b0b7 100644 --- a/internal/photos/models/file.go +++ b/internal/photos/models/file.go @@ -3,12 +3,13 @@ package models import "time" type File struct { - ID uint32 `gorm:"primary_key" json:"id"` - Name string `gorm:"not null" json:"name"` - Checksum string `gorm:"unique;size:44;not null"` - Size uint64 `gorm:"not null"` - Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"` - AuthorId *uint32 `json:"-"` + ID uint32 `gorm:"primary_key" json:"id"` + Name string `gorm:"not null" json:"name"` + Checksum string `gorm:"unique;size:44;not null"` + Size uint64 `gorm:"not null"` + Author *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"` + AuthorId *uint32 `json:"-"` + Chunks []*FileChunk CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` } diff --git a/internal/store/core.go b/internal/store/core.go index e7e8bf1..93c4654 100644 --- a/internal/store/core.go +++ b/internal/store/core.go @@ -34,7 +34,7 @@ func (s *Store) NewChunk(b []byte) *Chunk { } func (s *Store) LoadChunk(sum string) (*Chunk, error) { - c := &Chunk{s, sum, nil} + c := s.Chunk(sum) if !c.FileExists() { return nil, fmt.Errorf("chunk %s doesn't exists", sum) } @@ -46,9 +46,8 @@ func (s *Store) LoadChunk(sum string) (*Chunk, error) { return c, nil } -func (s *Store) ChunkExists(sum string) bool { - c := &Chunk{s, sum, nil} - return c.FileExists() +func (s *Store) Chunk(sum string) *Chunk { + return &Chunk{s, sum, nil} } func (c *Chunk) Dir() string { @@ -59,6 +58,14 @@ func (c *Chunk) Filename() string { return filepath.Join(c.Dir(), c.Sum) } +func (c *Chunk) Size() int64 { + st, err := os.Stat(c.Filename()) + if err != nil { + return -1 + } + return st.Size() +} + func (c *Chunk) FileExists() bool { fs, err := os.Stat(c.Filename()) if errors.Is(err, os.ErrNotExist) { diff --git a/internal/store/reader.go b/internal/store/reader.go new file mode 100644 index 0000000..306fa65 --- /dev/null +++ b/internal/store/reader.go @@ -0,0 +1,67 @@ +package store + +import ( + "io" + "os" + + "gitlab.celogeek.com/photos/api/internal/photoserrors" +) + +type StoreReaderChunk struct { + Filename string + Size int64 +} + +type StoreReader struct { + current *os.File + chunk int + chunks []StoreReaderChunk + Size int64 +} + +func (s *Store) NewStoreReader(chunks []string) (*StoreReader, error) { + sr := &StoreReader{nil, 0, make([]StoreReaderChunk, len(chunks)), 0} + for i, chunk := range chunks { + c := s.Chunk(chunk) + name := c.Filename() + size := c.Size() + if size < 0 { + return nil, photoserrors.ErrStoreMissingChunks + } + sr.chunks[i] = StoreReaderChunk{name, size} + sr.Size += size + } + return sr, nil +} + +func (s *StoreReader) Read(p []byte) (n int, err error) { + if s.current == nil { + f, err := os.Open(s.chunks[s.chunk].Filename) + if err != nil { + return -1, err + } + s.current = f + } + + n, err = s.current.Read(p) + if err == io.EOF { + s.chunk++ + if s.chunk > len(s.chunks)-1 { + return + } + s.Close() + return s.Read(p) + } + return +} + +func (s *StoreReader) Close() { + if s.current != nil { + s.current.Close() + s.current = nil + } +} + +// func (s *StoreReader) Seek(offset int64, whence int) (int64, error) { + +// }