reader multi file

cmd/photos-api-cli/upload.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/go-resty/resty/v2"
 	"github.com/schollz/progressbar/v3"
+	"gitlab.celogeek.com/photos/api/internal/photos/api"
 )
 
 type UploadCommand struct {
@@ -52,11 +53,6 @@ func (c *UploadCommand) FileExists() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	chunkSize := int64(1 << 20)
-	nbChunks := st.Size() / chunkSize
-	if st.Size()%chunkSize > 0 {
-		nbChunks++
-	}
 
 	progress := progressbar.DefaultBytes(st.Size(), fmt.Sprintf("Checking %s", filepath.Base(c.File)))
 	defer progress.Close()
@@ -86,9 +82,8 @@ func (c *UploadCommand) FileUpload(sum string) error {
 	if err != nil {
 		return err
 	}
-	chunkSize := int64(1 << 20)
-	nbChunks := st.Size() / chunkSize
-	if st.Size()%chunkSize > 0 {
+	nbChunks := st.Size() / api.CHUNK_SIZE
+	if st.Size()%api.CHUNK_SIZE > 0 {
 		nbChunks++
 	}
 
@@ -110,7 +105,7 @@ func (c *UploadCommand) FileUpload(sum string) error {
 	for w := uint32(0); w < c.Workers; w++ {
 		go func(w uint32) {
 			defer wg.Done()
-			b := make([]byte, chunkSize)
+			b := make([]byte, api.CHUNK_SIZE)
 			for {
 				mu.Lock()
 				part := i

internal/photos/api/file.go

@@ -3,8 +3,11 @@ package api
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
+	"mime"
 	"net/http"
+	"path/filepath"
 	"strings"
 
 	"github.com/gin-gonic/gin"
@@ -14,7 +17,7 @@ import (
 	"gorm.io/gorm"
 )
 
-var CHUNK_SIZE int64 = 1 << 20
+var CHUNK_SIZE int64 = 4 << 20
 
 type File struct {
 	Name     string   `json:"name"`
@@ -151,7 +154,7 @@ func (s *Service) FileChunkExists(c *gin.Context) {
 		return
 	}
 
-	if s.Store.ChunkExists(checksum) {
+	if s.Store.Chunk(checksum).FileExists() {
 		c.Status(http.StatusOK)
 	} else {
 		c.Status(http.StatusNotFound)
@@ -177,3 +180,44 @@ func (s *Service) FileExists(c *gin.Context) {
 		c.Status(http.StatusNotFound)
 	}
 }
+
+func (s *Service) FileGet(c *gin.Context) {
+	checksum := c.Param("checksum")
+	if len(checksum) != 40 {
+		s.Error(c, http.StatusBadRequest, photoserrors.ErrStoreBadChecksum)
+		return
+	}
+	if checksum == c.GetHeader("If-None-Match") {
+		c.Status(http.StatusNotModified)
+		return
+	}
+
+	f := &models.File{}
+	if err := s.DB.Debug().Preload("Chunks").Where("checksum = ?", checksum).First(&f).Error; err != nil {
+		s.Error(c, http.StatusBadRequest, err)
+		return
+	}
+
+	chunks := make([]string, len(f.Chunks))
+	for _, fc := range f.Chunks {
+		chunks[fc.Part-1] = fc.Checksum
+	}
+
+	reader, err := s.Store.NewStoreReader(chunks)
+	if err != nil {
+		s.Error(c, http.StatusInternalServerError, err)
+		return
+	}
+	defer reader.Close()
+
+	c.DataFromReader(
+		http.StatusOK,
+		reader.Size,
+		mime.TypeByExtension(filepath.Ext(f.Name)),
+		reader,
+		map[string]string{
+			"Content-Disposition": fmt.Sprintf("inline; filename=\"%s\"", f.Name),
+			"ETag":                f.Checksum,
+		},
+	)
+}

internal/photos/api/main.go

@@ -61,6 +61,7 @@ func (s *Service) SetupRoutes() {
 	album.Use(s.RequireSession)
 	album.POST("", s.FileCreate)
 	album.HEAD("/:checksum", s.FileExists)
+	album.GET("/:checksum", s.FileGet)
 
 	album.POST("/chunk", s.FileCreateChunk)
 	album.HEAD("/chunk/:checksum", s.FileChunkExists)

internal/photos/api/session.go

@@ -13,13 +13,20 @@ import (
 )
 
 func (s *Service) RequireAuthToken(c *gin.Context) {
-	token := c.GetHeader("Authorization")
-	if !strings.HasPrefix(token, "Private ") {
+	tokenAuth := c.GetHeader("Authorization")
+	tokenCookie, _ := c.Cookie("photoapitoken")
+
+	if tokenAuth != "" {
+		if !strings.HasPrefix(tokenAuth, "Private ") {
+			s.Error(c, http.StatusForbidden, photoserrors.ErrTokenMissing)
+		} else {
+			c.Set("token", tokenAuth[8:])
+		}
+	} else if tokenCookie != "" {
+		c.Set("token", tokenCookie)
+	} else {
 		s.Error(c, http.StatusForbidden, photoserrors.ErrTokenMissing)
-		return
 	}
-	token = token[8:]
-	c.Set("token", token)
 }
 
 func (s *Service) RequireSession(c *gin.Context) {

internal/photos/models/file.go

@@ -3,12 +3,13 @@ package models
 import "time"
 
 type File struct {
-	ID        uint32    `gorm:"primary_key" json:"id"`
-	Name      string    `gorm:"not null" json:"name"`
-	Checksum  string    `gorm:"unique;size:44;not null"`
-	Size      uint64    `gorm:"not null"`
-	Author    *Account  `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"`
-	AuthorId  *uint32   `json:"-"`
+	ID        uint32   `gorm:"primary_key" json:"id"`
+	Name      string   `gorm:"not null" json:"name"`
+	Checksum  string   `gorm:"unique;size:44;not null"`
+	Size      uint64   `gorm:"not null"`
+	Author    *Account `gorm:"constraint:OnDelete:SET NULL,OnUpdate:CASCADE" json:"author"`
+	AuthorId  *uint32  `json:"-"`
+	Chunks    []*FileChunk
 	CreatedAt time.Time `json:"created_at"`
 	UpdatedAt time.Time `json:"updated_at"`
 }

internal/store/core.go

@@ -34,7 +34,7 @@ func (s *Store) NewChunk(b []byte) *Chunk {
 }
 
 func (s *Store) LoadChunk(sum string) (*Chunk, error) {
-	c := &Chunk{s, sum, nil}
+	c := s.Chunk(sum)
 	if !c.FileExists() {
 		return nil, fmt.Errorf("chunk %s doesn't exists", sum)
 	}
@@ -46,9 +46,8 @@ func (s *Store) LoadChunk(sum string) (*Chunk, error) {
 	return c, nil
 }
 
-func (s *Store) ChunkExists(sum string) bool {
-	c := &Chunk{s, sum, nil}
-	return c.FileExists()
+func (s *Store) Chunk(sum string) *Chunk {
+	return &Chunk{s, sum, nil}
 }
 
 func (c *Chunk) Dir() string {
@@ -59,6 +58,14 @@ func (c *Chunk) Filename() string {
 	return filepath.Join(c.Dir(), c.Sum)
 }
 
+func (c *Chunk) Size() int64 {
+	st, err := os.Stat(c.Filename())
+	if err != nil {
+		return -1
+	}
+	return st.Size()
+}
+
 func (c *Chunk) FileExists() bool {
 	fs, err := os.Stat(c.Filename())
 	if errors.Is(err, os.ErrNotExist) {

internal/store/reader.go

@@ -0,0 +1,67 @@
+package store
+
+import (
+	"io"
+	"os"
+
+	"gitlab.celogeek.com/photos/api/internal/photoserrors"
+)
+
+type StoreReaderChunk struct {
+	Filename string
+	Size     int64
+}
+
+type StoreReader struct {
+	current *os.File
+	chunk   int
+	chunks  []StoreReaderChunk
+	Size    int64
+}
+
+func (s *Store) NewStoreReader(chunks []string) (*StoreReader, error) {
+	sr := &StoreReader{nil, 0, make([]StoreReaderChunk, len(chunks)), 0}
+	for i, chunk := range chunks {
+		c := s.Chunk(chunk)
+		name := c.Filename()
+		size := c.Size()
+		if size < 0 {
+			return nil, photoserrors.ErrStoreMissingChunks
+		}
+		sr.chunks[i] = StoreReaderChunk{name, size}
+		sr.Size += size
+	}
+	return sr, nil
+}
+
+func (s *StoreReader) Read(p []byte) (n int, err error) {
+	if s.current == nil {
+		f, err := os.Open(s.chunks[s.chunk].Filename)
+		if err != nil {
+			return -1, err
+		}
+		s.current = f
+	}
+
+	n, err = s.current.Read(p)
+	if err == io.EOF {
+		s.chunk++
+		if s.chunk > len(s.chunks)-1 {
+			return
+		}
+		s.Close()
+		return s.Read(p)
+	}
+	return
+}
+
+func (s *StoreReader) Close() {
+	if s.current != nil {
+		s.current.Close()
+		s.current = nil
+	}
+}
+
+// func (s *StoreReader) Seek(offset int64, whence int) (int64, error) {
+
+// }