From ab15c8c5a897e47bcf227534a1cb593aa4de1905 Mon Sep 17 00:00:00 2001 From: celogeek Date: Sat, 5 Mar 2022 17:03:53 +0100 Subject: [PATCH] upload file --- cmd/photos-api-cli/login.go | 8 +-- cmd/photos-api-cli/register.go | 8 +-- cmd/photos-api-cli/upload.go | 107 +++++++++++++++++++++++++++++++++ go.mod | 13 ++-- go.sum | 25 ++++++-- internal/photos/api/file.go | 21 ++++++- internal/photos/api/main.go | 2 +- 7 files changed, 162 insertions(+), 22 deletions(-) create mode 100644 cmd/photos-api-cli/upload.go diff --git a/cmd/photos-api-cli/login.go b/cmd/photos-api-cli/login.go index 726b48e..c029b7e 100644 --- a/cmd/photos-api-cli/login.go +++ b/cmd/photos-api-cli/login.go @@ -13,8 +13,6 @@ type LoginCommand struct { Password string `short:"p" long:"password" description:"Password" required:"true"` } -var loginCommand LoginCommand - type LoginRequest struct { Login string `json:"login"` Password string `json:"password"` @@ -31,11 +29,11 @@ type LoginResponse struct { func (c *LoginCommand) Execute(args []string) error { logger.Printf("Login on %s...\n", c.Url) - cli := resty.New().SetBaseURL(loginCommand.Url) + cli := resty.New().SetBaseURL(c.Url) resp, err := cli. R(). - SetBody(&LoginRequest{loginCommand.Login, loginCommand.Password}). + SetBody(&LoginRequest{c.Login, c.Password}). SetResult(&LoginResponse{}). SetError(&LoginError{}). Post("/account/login") @@ -58,5 +56,5 @@ func (c *LoginCommand) Execute(args []string) error { } func init() { - parser.AddCommand("login", "Login", "", &loginCommand) + parser.AddCommand("login", "Login", "", &LoginCommand{}) } diff --git a/cmd/photos-api-cli/register.go b/cmd/photos-api-cli/register.go index ea748f2..5438730 100644 --- a/cmd/photos-api-cli/register.go +++ b/cmd/photos-api-cli/register.go @@ -12,8 +12,6 @@ type RegisterCommand struct { Password string `short:"p" long:"password" description:"Password" required:"true"` } -var registerCommand RegisterCommand - type RegisterRequest struct { Login string `json:"login"` Password string `json:"password"` @@ -29,11 +27,11 @@ type RegisterError struct { func (c *RegisterCommand) Execute(args []string) error { logger.Printf("Registering on %s...\n", c.Url) - cli := resty.New().SetBaseURL(registerCommand.Url) + cli := resty.New().SetBaseURL(c.Url) resp, err := cli. R(). - SetBody(&RegisterRequest{registerCommand.Login, registerCommand.Password}). + SetBody(&RegisterRequest{c.Login, c.Password}). SetResult(&RegisterResponse{}). SetError(&RegisterError{}). Post("/account/signup") @@ -53,5 +51,5 @@ func (c *RegisterCommand) Execute(args []string) error { } func init() { - parser.AddCommand("register", "Register", "", ®isterCommand) + parser.AddCommand("register", "Register", "", &RegisterCommand{}) } diff --git a/cmd/photos-api-cli/upload.go b/cmd/photos-api-cli/upload.go new file mode 100644 index 0000000..0123e24 --- /dev/null +++ b/cmd/photos-api-cli/upload.go @@ -0,0 +1,107 @@ +package main + +import ( + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/go-resty/resty/v2" + "github.com/schollz/progressbar/v3" +) + +type UploadCommand struct { + Url string `short:"u" long:"url" description:"Url of the instance" required:"true"` + Token string `short:"t" long:"token" description:"Token of the instance" required:"true"` + File string `short:"f" long:"file" description:"File to upload" required:"true"` +} + +type UploadError struct { + Error string +} + +type UploadChunkSuccess struct { + Checksum string +} + +type UploadFileRequest struct { + Name string + Checksum string + Chunks []string +} + +type UploadFileResponse struct { + Sum string + NbChunks uint32 + Size uint64 +} + +func (c *UploadCommand) Execute(args []string) error { + f, err := os.Open(c.File) + if err != nil { + return err + } + defer f.Close() + + st, err := f.Stat() + if err != nil { + return err + } + + uploadFile := &UploadFileRequest{Name: filepath.Base(c.File)} + progress := progressbar.DefaultBytes(st.Size(), fmt.Sprintf("Uploading %s", uploadFile.Name)) + defer progress.Close() + + cli := resty.New().SetBaseURL(c.Url).SetAuthScheme("Private").SetAuthToken(c.Token) + b := make([]byte, 1<<20) + checksum := sha1.New() + for { + n, err := f.Read(b) + if n == 0 { + if err == io.EOF { + break + } + return err + } + + resp, err := cli.R().SetError(&UploadError{}).SetResult(&UploadChunkSuccess{}).SetBody(b[0:n]).Post("/file/chunk") + if err != nil { + return err + } + + if err, ok := resp.Error().(*UploadError); ok { + return errors.New(err.Error) + } + + if result, ok := resp.Result().(*UploadChunkSuccess); ok { + uploadFile.Chunks = append(uploadFile.Chunks, result.Checksum) + checksum.Write(b[0:n]) + progress.Add(n) + } + } + + uploadFile.Checksum = hex.EncodeToString(checksum.Sum(nil)) + + resp, err := cli.R().SetBody(uploadFile).SetError(&UploadError{}).SetResult(&UploadFileResponse{}).Post("/file") + if err != nil { + return err + } + + if err, ok := resp.Error().(*UploadError); ok { + logger.Println("Upload failed") + return errors.New(err.Error) + } + + if result, ok := resp.Result().(*UploadFileResponse); ok { + fmt.Printf("Upload succeed\nSum: %s\nNbChunks: %d\nSize: %d\n", result.Sum, result.NbChunks, result.Size) + } + + return nil +} + +func init() { + parser.AddCommand("upload", "Upload a file", "", &UploadCommand{}) +} diff --git a/go.mod b/go.mod index e108773..572e2fc 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/go-sql-driver/mysql v1.6.0 github.com/google/uuid v1.3.0 github.com/jessevdk/go-flags v1.5.0 + github.com/schollz/progressbar/v3 v3.8.6 gopkg.in/validator.v2 v2.0.0-20210331031555-b37d688a7fb0 gorm.io/driver/mysql v1.2.3 gorm.io/gorm v1.22.5 @@ -23,12 +24,16 @@ require ( github.com/jinzhu/now v1.1.4 // indirect github.com/json-iterator/go v1.1.9 // indirect github.com/leodido/go-urn v1.2.0 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/ugorji/go/codec v1.1.7 // indirect - golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect - golang.org/x/net v0.0.0-20211029224645-99673261e6eb // indirect - golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect + golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect ) diff --git a/go.sum b/go.sum index f3d8144..67322fe 100644 --- a/go.sum +++ b/go.sum @@ -31,16 +31,26 @@ github.com/jinzhu/now v1.1.4 h1:tHnRBy1i5F2Dh8BAFxqFzxKqqvezXrL2OW1TnX+Mlas= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/schollz/progressbar/v3 v3.8.6 h1:QruMUdzZ1TbEP++S1m73OqRJk20ON11m6Wqv4EoGg8c= +github.com/schollz/progressbar/v3 v3.8.6/go.mod h1:W5IEwbJecncFGBvuEh4A7HT1nZZ6WNIL2i3qbnI0WKY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= @@ -50,19 +60,26 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838 h1:71vQrMauZZhcTVK6KdYM+rklehEEwb3E+ZhaE5jrPrE= +golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20211029224645-99673261e6eb h1:pirldcYWx7rx7kE5r+9WsOXPXK0+WH5+uZ7uPmJ44uM= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 h1:XDXtA5hveEEV8JB2l7nhMTp3t3cHp9ZpwcdjqyEWLlo= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/internal/photos/api/file.go b/internal/photos/api/file.go index acaebe9..f3b9a1d 100644 --- a/internal/photos/api/file.go +++ b/internal/photos/api/file.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gin-gonic/gin" + "github.com/go-sql-driver/mysql" "gitlab.celogeek.com/photos/api/internal/photos/models" "gitlab.celogeek.com/photos/api/internal/photoserrors" "gorm.io/gorm" @@ -90,6 +91,15 @@ func (s *Service) FileCreate(c *gin.Context) { return nil }) + if nerr, ok := err.(*mysql.MySQLError); ok { + if nerr.Number == 1062 { + // duplicate error + if strings.HasSuffix(nerr.Message, "for key 'checksum'") { + err = nil + } + } + } + if err != nil { s.Error(c, http.StatusInternalServerError, err) return @@ -115,9 +125,13 @@ func (s *Service) FileCreateChunk(c *gin.Context) { sess := s.CurrentSession(c) - if err := s.Store.NewChunk(b.Bytes()).Save(sess); err != nil { + chunk := s.Store.NewChunk(b.Bytes()) + if err := chunk.Save(sess); err != nil { if errors.Is(err, photoserrors.ErrStoreChunkAlreadyExists) { - s.Error(c, http.StatusOK, err) + c.JSON(http.StatusOK, gin.H{ + "status": "success", + "checksum": chunk.Sum, + }) } else { s.Error(c, http.StatusBadRequest, err) } @@ -125,6 +139,7 @@ func (s *Service) FileCreateChunk(c *gin.Context) { } c.JSON(http.StatusOK, gin.H{ - "status": "success", + "status": "success", + "checksum": chunk.Sum, }) } diff --git a/internal/photos/api/main.go b/internal/photos/api/main.go index 21f2027..71bfb7e 100644 --- a/internal/photos/api/main.go +++ b/internal/photos/api/main.go @@ -59,7 +59,7 @@ func (s *Service) SetupRoutes() { album := s.Gin.Group("/file") album.Use(s.RequireSession) - album.POST("/", s.FileCreate) + album.POST("", s.FileCreate) album.POST("/chunk", s.FileCreateChunk) s.Gin.NoRoute(func(c *gin.Context) {