diff --git a/internal/piwigo/file_to_upload.go b/internal/piwigo/file_to_upload.go new file mode 100644 index 0000000..323137c --- /dev/null +++ b/internal/piwigo/file_to_upload.go @@ -0,0 +1,57 @@ +package piwigo + +import ( + "os" + "path/filepath" + "strings" +) + +type FileUploadResult struct { + ImageId int `json:"image_id"` + Url string `json:"url"` +} + +type FileToUpload struct { + Dir string + Name string + CategoryId int + + md5 *string + size *int64 + ext *string +} + +func (f *FileToUpload) FullPath() string { + return filepath.Join(f.Dir, f.Name) +} + +func (f *FileToUpload) MD5() string { + if f.md5 == nil { + md5, err := Md5File(f.FullPath()) + if err != nil { + return "" + } + f.md5 = &md5 + } + return *f.md5 +} + +func (f *FileToUpload) Size() int64 { + if f.size == nil { + st, err := os.Stat(f.FullPath()) + if err != nil { + return -1 + } + size := st.Size() + f.size = &size + } + return *f.size +} + +func (f *FileToUpload) Ext() string { + if f.ext == nil { + ext := strings.ToLower(filepath.Ext(f.Name)[1:]) + f.ext = &ext + } + return *f.ext +} diff --git a/internal/piwigo/file_to_upload_stat.go b/internal/piwigo/file_to_upload_stat.go new file mode 100644 index 0000000..2219702 --- /dev/null +++ b/internal/piwigo/file_to_upload_stat.go @@ -0,0 +1,72 @@ +package piwigo + +import ( + "fmt" + "sync" + + "github.com/schollz/progressbar/v3" +) + +type FileToUploadStat struct { + Checked int64 + Total int64 + TotalBytes int64 + Uploaded int64 + UploadedBytes int64 + Skipped int64 + Failed int64 + Progress *progressbar.ProgressBar + mu sync.Mutex +} + +func (s *FileToUploadStat) Refresh() { + s.Progress.Describe(fmt.Sprintf("check:%d, upload:%d, skip:%d, failed:%d, total:%d", s.Checked, s.Uploaded, s.Skipped, s.Failed, s.Total)) +} + +func (s *FileToUploadStat) Check() { + s.mu.Lock() + s.Checked++ + s.Refresh() + s.mu.Unlock() +} + +func (s *FileToUploadStat) Add(filesize int64) { + s.mu.Lock() + s.Total++ + s.TotalBytes += filesize + s.Progress.ChangeMax64(s.TotalBytes) + s.Refresh() + s.mu.Unlock() +} + +func (s *FileToUploadStat) Commit(filereaded int64) { + s.mu.Lock() + s.UploadedBytes += filereaded + s.Progress.Set64(s.UploadedBytes) + s.mu.Unlock() +} + +func (s *FileToUploadStat) Done() { + s.mu.Lock() + s.Uploaded++ + s.Refresh() + s.mu.Unlock() +} + +func (s *FileToUploadStat) Close() { + s.Progress.Close() +} + +func (s *FileToUploadStat) Fail() { + s.mu.Lock() + s.Failed++ + s.Refresh() + s.mu.Unlock() +} + +func (s *FileToUploadStat) Skip() { + s.mu.Lock() + s.Skipped++ + s.Refresh() + s.mu.Unlock() +} diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go index 74d0177..b1a3515 100644 --- a/internal/piwigo/files.go +++ b/internal/piwigo/files.go @@ -5,20 +5,13 @@ import ( "fmt" "io/ioutil" "net/url" - "os" "path/filepath" "strings" "sync" - "github.com/schollz/progressbar/v3" "golang.org/x/text/unicode/norm" ) -type FileUploadResult struct { - ImageId int `json:"image_id"` - Url string `json:"url"` -} - func (p *Piwigo) FileExists(md5 string) bool { var resp map[string]*string @@ -31,35 +24,32 @@ func (p *Piwigo) FileExists(md5 string) bool { return resp[md5] != nil } -func (p *Piwigo) UploadChunks(filename string, nbJobs int, categoryId int) (*FileUploadResult, error) { - md5, err := Md5File(filename, false) - if err != nil { - return nil, err +func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, hasVideoJS bool) error { + if file.MD5() == "" { + stat.Fail() + return errors.New("checksum error") } - if p.FileExists(md5) { - return nil, errors.New("file already exists") + stat.Check() + + if p.FileExists(file.MD5()) { + stat.Skip() + return errors.New("file already exists") } - st, _ := os.Stat(filename) wg := &sync.WaitGroup{} - chunks, err := Base64Chunker(filename) + chunks, err := Base64Chunker(file.FullPath()) errout := make(chan error) - bar := progressbar.DefaultBytes( - st.Size(), - "uploading", - ) if err != nil { - return nil, err + return err } for j := 0; j < nbJobs; j++ { wg.Add(1) - go p.UploadChunk(md5, chunks, errout, wg, bar) + go p.UploadChunk(file.MD5(), chunks, errout, wg, stat) } go func() { wg.Wait() - bar.Close() close(errout) }() @@ -68,30 +58,40 @@ func (p *Piwigo) UploadChunks(filename string, nbJobs int, categoryId int) (*Fil errstring += err.Error() + "\n" } if errstring != "" { - return nil, errors.New(errstring) + stat.Fail() + return errors.New(errstring) } - exif, _ := Exif(filename) + exif, _ := Exif(file.FullPath()) var resp *FileUploadResult data := &url.Values{} - data.Set("original_sum", md5) - data.Set("original_filename", filepath.Base(filename)) + data.Set("original_sum", file.MD5()) + data.Set("original_filename", file.Name) data.Set("check_uniqueness", "true") if exif != nil && exif.CreatedAt != nil { data.Set("date_creation", exif.CreatedAt.String()) } - if categoryId > 0 { - data.Set("categories", fmt.Sprint(categoryId)) + if file.CategoryId > 0 { + data.Set("categories", fmt.Sprint(file.CategoryId)) } err = p.Post("pwg.images.add", data, &resp) if err != nil { - return nil, err + stat.Fail() + return err } - return resp, nil + if hasVideoJS { + switch file.Ext() { + case "ogg", "ogv", "mp4", "m4v", "webm", "webmv": + p.VideoJSSync(resp.ImageId) + } + } + + stat.Done() + return nil } -func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, bar *progressbar.ProgressBar) { +func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, progress *FileToUploadStat) { defer wg.Done() for chunk := range chunks { var err error @@ -107,7 +107,7 @@ func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout break } } - bar.Add64(chunk.Size) + progress.Commit(chunk.Size) if err != nil { errout <- fmt.Errorf("error on chunk %d: %v", chunk.Position, err) continue @@ -115,12 +115,6 @@ func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout } } -type FileToUpload struct { - Filename string - Md5 string - CategoryId int -} - func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, filter UploadFileType) ([]FileToUpload, error) { rootPath, err := filepath.Abs(rootPath) if err != nil { @@ -147,7 +141,7 @@ func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, fi continue } filename := filepath.Join(rootPath, dir.Name()) - md5, err := Md5File(filename, false) + md5, err := Md5File(filename) if err != nil { return nil, err } @@ -158,8 +152,8 @@ func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, fi fmt.Printf("%s - %s %s - %s\n", levelStr, dir.Name(), md5, status) if status == "OK" { files = append(files, FileToUpload{ - Filename: filename, - Md5: md5, + Dir: rootPath, + Name: dir.Name(), CategoryId: parentCategoryId, }) } diff --git a/internal/piwigo/helper.go b/internal/piwigo/helper.go index cf8a60d..8041f1a 100644 --- a/internal/piwigo/helper.go +++ b/internal/piwigo/helper.go @@ -11,8 +11,6 @@ import ( "net/url" "os" "strings" - - "github.com/schollz/progressbar/v3" ) var CHUNK_BUFF_SIZE int64 = 32 * 1024 @@ -39,26 +37,20 @@ func ArgsToForm(args []string) (*url.Values, error) { return params, nil } -func Md5File(filename string, progress bool) (string, error) { +func Md5File(filename string) (result string, err error) { file, err := os.Open(filename) if err != nil { - return "", err + return } defer file.Close() - st, _ := file.Stat() hash := md5.New() - - if progress { - bar := progressbar.DefaultBytes(st.Size(), "checksumming") - _, err = io.Copy(io.MultiWriter(hash, bar), file) - } else { - _, err = io.Copy(hash, file) - } + _, err = io.Copy(hash, file) if err != nil { - return "", err + return } - return fmt.Sprintf("%x", hash.Sum(nil)), nil + result = fmt.Sprintf("%x", hash.Sum(nil)) + return } type Base64ChunkResult struct { diff --git a/internal/piwigocli/images_upload.go b/internal/piwigocli/images_upload.go index d0d1d63..85fb026 100644 --- a/internal/piwigocli/images_upload.go +++ b/internal/piwigocli/images_upload.go @@ -2,11 +2,11 @@ package piwigocli import ( "errors" - "fmt" "path/filepath" "strings" "github.com/celogeek/piwigo-cli/internal/piwigo" + "github.com/schollz/progressbar/v3" ) type ImagesUploadCommand struct { @@ -31,21 +31,25 @@ func (c *ImagesUploadCommand) Execute(args []string) error { return errors.New("unsupported file extension") } - resp, err := p.UploadChunks(c.Filename, c.NbJobs, c.CategoryId) + _, hasVideoJS := status.Plugins["piwigo-videojs"] + + file := &piwigo.FileToUpload{ + Dir: filepath.Dir(c.Filename), + Name: filepath.Base(c.Filename), + CategoryId: c.CategoryId, + } + + stat := &piwigo.FileToUploadStat{ + Progress: progressbar.DefaultBytes(1, "prepare"), + } + defer stat.Close() + stat.Add(file.Size()) + stat.Refresh() + + err = p.Upload(file, stat, c.NbJobs, hasVideoJS) if err != nil { return err } - if _, ok := status.Plugins["piwigo-videojs"]; ok { - switch ext { - case "ogg", "ogv", "mp4", "m4v", "webm", "webmv": - fmt.Println("syncing metadata with videojs") - err = p.VideoJSSync(resp.ImageId) - if err != nil { - return err - } - } - } - return nil }