diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go index d58319c..db3a1e1 100644 --- a/internal/piwigo/files.go +++ b/internal/piwigo/files.go @@ -10,6 +10,11 @@ import ( "github.com/schollz/progressbar/v3" ) +type FileUploadResult struct { + ImageId int `json:"image_id"` + Url string `json:"url"` +} + func (p *Piwigo) FileExists(md5 string) bool { var resp map[string]*string @@ -22,83 +27,79 @@ func (p *Piwigo) FileExists(md5 string) bool { return resp[md5] != nil } -func (p *Piwigo) UploadChunks(filename string, nbJobs int) error { +func (p *Piwigo) UploadChunks(filename string, nbJobs int) (*FileUploadResult, error) { md5, err := Md5File(filename) if err != nil { - return err + return nil, err } if p.FileExists(md5) { - return errors.New("file already exists") + return nil, errors.New("file already exists") } + st, _ := os.Stat(filename) - f, err := os.Open(filename) - if err != nil { - return err - } - defer f.Close() - - st, err := f.Stat() - if err != nil { - return err - } - - in := make(chan int64) - out := make(chan error) wg := &sync.WaitGroup{} + chunks, err := Base64Chunker(filename) + errout := make(chan error) bar := progressbar.DefaultBytes( st.Size(), "uploading", ) + if err != nil { + return nil, err + } for j := 0; j < nbJobs; j++ { wg.Add(1) - go p.UploadChunk(md5, f, in, out, wg, bar) + go p.UploadChunk(md5, chunks, errout, wg, bar) } - go func() { - nbChunks := st.Size()/CHUNK_SIZE + 1 - for position := int64(0); position < nbChunks; position++ { - in <- position - } - close(in) wg.Wait() - close(out) bar.Close() + close(errout) }() - var errString string - for err := range out { - errString += err.Error() + "\n" + var errstring string + for err := range errout { + errstring += err.Error() + "\n" } - if errString != "" { - return errors.New(errString[:len(errString)-1]) + if errstring != "" { + return nil, errors.New(errstring) } - fmt.Println(md5) + var resp *FileUploadResult + err = p.Post("pwg.images.add", &url.Values{ + "original_sum": []string{md5}, + "original_filename": []string{filename}, + "check_uniqueness": []string{"true"}, + }, &resp) + if err != nil { + return nil, err + } - return nil + return resp, nil } -func (p *Piwigo) UploadChunk(md5 string, f *os.File, in chan int64, out chan error, wg *sync.WaitGroup, bar *progressbar.ProgressBar) { +func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, bar *progressbar.ProgressBar) { defer wg.Done() - for position := range in { - n, b64, err := Base64Chunk(f, position) - if err != nil { - out <- fmt.Errorf("error on chunk %d: %v", position, err) - continue - } - - err = p.Post("pwg.images.addChunk", &url.Values{ + for chunk := range chunks { + var err error + data := &url.Values{ "original_sum": []string{md5}, - "position": []string{fmt.Sprint(position)}, + "position": []string{fmt.Sprint(chunk.Position)}, "type": []string{"file"}, - "data": []string{b64}, - }, nil) + "data": []string{chunk.Buffer.String()}, + } + for i := 0; i < 3; i++ { + err = p.Post("pwg.images.addChunk", data, nil) + if err == nil { + break + } + } + bar.Add64(chunk.Size) if err != nil { - out <- fmt.Errorf("error on chunk %d: %v", position, err) + errout <- fmt.Errorf("error on chunk %d: %v", chunk.Position, err) continue } - bar.Add(n) } } diff --git a/internal/piwigo/helper.go b/internal/piwigo/helper.go index 90e7261..41aed35 100644 --- a/internal/piwigo/helper.go +++ b/internal/piwigo/helper.go @@ -1,19 +1,23 @@ package piwigo import ( + "bytes" "crypto/md5" "encoding/base64" "encoding/json" "errors" "fmt" "io" - "math" "net/url" "os" "strings" + + "github.com/schollz/progressbar/v3" ) -var CHUNK_SIZE int64 = int64(math.Pow(1024, 2)) +var CHUNK_BUFF_SIZE int64 = 32 * 1024 +var CHUNK_BUFF_COUNT int = 32 +var CHUNK_PRECOMPUTE_SIZE int = 8 func DumpResponse(v interface{}) (err error) { b, err := json.MarshalIndent(v, "", " ") @@ -40,22 +44,55 @@ func Md5File(filename string) (string, error) { if err != nil { return "", err } + defer file.Close() + + st, _ := file.Stat() + bar := progressbar.DefaultBytes(st.Size(), "checksumming") + hash := md5.New() - _, err = io.Copy(hash, file) + _, err = io.Copy(io.MultiWriter(hash, bar), file) if err != nil { return "", err } return fmt.Sprintf("%x", hash.Sum(nil)), nil } -func Base64Chunk(file *os.File, position int64) (int, string, error) { - b := make([]byte, CHUNK_SIZE) - n, err := file.ReadAt(b, position*CHUNK_SIZE) - if err != nil && err != io.EOF { - return 0, "", err - } - if n == 0 { - return 0, "", errors.New("position out of bound") - } - return n, base64.StdEncoding.EncodeToString(b[:n]), nil +type Base64ChunkResult struct { + Position int64 + Size int64 + Buffer bytes.Buffer +} + +func Base64Chunker(filename string) (out chan *Base64ChunkResult, err error) { + f, err := os.Open(filename) + if err != nil { + return + } + + out = make(chan *Base64ChunkResult, CHUNK_PRECOMPUTE_SIZE) + go func() { + b := make([]byte, CHUNK_BUFF_SIZE) + defer f.Close() + defer close(out) + ok := false + for position := int64(0); !ok; position += 1 { + bf := &Base64ChunkResult{ + Position: position, + } + b64 := base64.NewEncoder(base64.StdEncoding, &bf.Buffer) + for i := 0; i < CHUNK_BUFF_COUNT; i++ { + n, _ := f.Read(b) + if n == 0 { + ok = true + break + } + bf.Size += int64(n) + b64.Write(b[:n]) + } + b64.Close() + out <- bf + } + }() + + return } diff --git a/internal/piwigo/login.go b/internal/piwigo/login.go index 64ea7f4..7bc049d 100644 --- a/internal/piwigo/login.go +++ b/internal/piwigo/login.go @@ -1,14 +1,44 @@ package piwigo import ( + "encoding/json" "errors" "net/url" + "strings" ) +type UploadFileType map[string]bool + type StatusResponse struct { - User string `json:"username"` - Role string `json:"status"` - Version string `json:"version"` + User string `json:"username"` + Role string `json:"status"` + Version string `json:"version"` + Token string `json:"pwg_token"` + UploadFileType UploadFileType `json:"upload_file_types"` +} + +func (uft *UploadFileType) UnmarshalJSON(data []byte) error { + var r string + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *uft = UploadFileType{} + for _, v := range strings.Split(r, ",") { + (*uft)[v] = true + } + return nil +} + +func (uft UploadFileType) MarshalJSON() ([]byte, error) { + return []byte(`"` + uft.String() + `"`), nil +} + +func (uft UploadFileType) String() string { + keys := make([]string, 0, len(uft)) + for k, _ := range uft { + keys = append(keys, k) + } + return strings.Join(keys, ",") } func (p *Piwigo) GetStatus() (*StatusResponse, error) { @@ -22,7 +52,6 @@ func (p *Piwigo) GetStatus() (*StatusResponse, error) { if err != nil { return nil, err } - if resp.User == p.Username { return resp, nil } diff --git a/internal/piwigo/post.go b/internal/piwigo/post.go index 8423b82..94fa4ec 100644 --- a/internal/piwigo/post.go +++ b/internal/piwigo/post.go @@ -61,7 +61,7 @@ func (p *Piwigo) Post(method string, form *url.Values, resp interface{}) error { newBody := &bytes.Buffer{} tee := io.TeeReader(r.Body, newBody) - var RawResult map[string]interface{} + var RawResult interface{} err = json.NewDecoder(tee).Decode(&RawResult) if err != nil { return err diff --git a/internal/piwigo/videojs.go b/internal/piwigo/videojs.go new file mode 100644 index 0000000..070d8c5 --- /dev/null +++ b/internal/piwigo/videojs.go @@ -0,0 +1,36 @@ +package piwigo + +import ( + "fmt" + "net/http" + "net/url" +) + +func (p *Piwigo) VideoJSSync(imageId int) error { + Url, err := url.Parse(p.Url) + if err != nil { + return err + } + Url.Path += "/admin.php" + q := Url.Query() + q.Set("page", "plugin") + q.Set("section", "piwigo-videojs/admin/admin_photo.php") + q.Set("sync_metadata", "1") + q.Set("image_id", fmt.Sprint(imageId)) + Url.RawQuery = q.Encode() + + req, err := http.NewRequest("GET", Url.String(), nil) + if err != nil { + return err + } + if p.Token != "" { + req.AddCookie(&http.Cookie{Name: "pwg_id", Value: p.Token, HttpOnly: true}) + } + + r, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer r.Body.Close() + return nil +} diff --git a/internal/piwigocli/images_upload.go b/internal/piwigocli/images_upload.go index 49edc4c..e5e0ac9 100644 --- a/internal/piwigocli/images_upload.go +++ b/internal/piwigocli/images_upload.go @@ -1,6 +1,10 @@ package piwigocli import ( + "errors" + "path/filepath" + "strings" + "github.com/celogeek/piwigo-cli/internal/piwigo" ) @@ -15,15 +19,28 @@ func (c *ImagesUploadCommand) Execute(args []string) error { return err } - _, err := p.Login() + status, err := p.Login() if err != nil { return err } - err = p.UploadChunks(c.Filename, c.NBJobs) + ext := strings.ToLower(filepath.Ext(c.Filename)[1:]) + if _, ok := status.UploadFileType[ext]; !ok { + return errors.New("unsupported file extension") + } + + resp, err := p.UploadChunks(c.Filename, c.NBJobs) if err != nil { return err } + switch ext { + case "ogg", "ogv", "mp4", "m4v", "webm", "webmv": + err = p.VideoJSSync(resp.ImageId) + if err != nil { + return err + } + } + return nil } diff --git a/internal/piwigocli/session_status.go b/internal/piwigocli/session_status.go index b38432b..61a6ace 100644 --- a/internal/piwigocli/session_status.go +++ b/internal/piwigocli/session_status.go @@ -28,6 +28,8 @@ func (c *SessionStatusCommand) Execute(args []string) error { {"Version", resp.Version}, {"User", resp.User}, {"Role", resp.Role}, + {"Admin Token", resp.Token}, + {"Supported formats", resp.UploadFileType}, }) t.SetOutputMirror(os.Stdout)