improve progress

internal/piwigo/file_to_upload.go

@@ -0,0 +1,57 @@
+package piwigo
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type FileUploadResult struct {
+	ImageId int    `json:"image_id"`
+	Url     string `json:"url"`
+}
+
+type FileToUpload struct {
+	Dir        string
+	Name       string
+	CategoryId int
+
+	md5  *string
+	size *int64
+	ext  *string
+}
+
+func (f *FileToUpload) FullPath() string {
+	return filepath.Join(f.Dir, f.Name)
+}
+
+func (f *FileToUpload) MD5() string {
+	if f.md5 == nil {
+		md5, err := Md5File(f.FullPath())
+		if err != nil {
+			return ""
+		}
+		f.md5 = &md5
+	}
+	return *f.md5
+}
+
+func (f *FileToUpload) Size() int64 {
+	if f.size == nil {
+		st, err := os.Stat(f.FullPath())
+		if err != nil {
+			return -1
+		}
+		size := st.Size()
+		f.size = &size
+	}
+	return *f.size
+}
+
+func (f *FileToUpload) Ext() string {
+	if f.ext == nil {
+		ext := strings.ToLower(filepath.Ext(f.Name)[1:])
+		f.ext = &ext
+	}
+	return *f.ext
+}

internal/piwigo/file_to_upload_stat.go

@@ -0,0 +1,72 @@
+package piwigo
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/schollz/progressbar/v3"
+)
+
+type FileToUploadStat struct {
+	Checked       int64
+	Total         int64
+	TotalBytes    int64
+	Uploaded      int64
+	UploadedBytes int64
+	Skipped       int64
+	Failed        int64
+	Progress      *progressbar.ProgressBar
+	mu            sync.Mutex
+}
+
+func (s *FileToUploadStat) Refresh() {
+	s.Progress.Describe(fmt.Sprintf("check:%d, upload:%d, skip:%d, failed:%d, total:%d", s.Checked, s.Uploaded, s.Skipped, s.Failed, s.Total))
+}
+
+func (s *FileToUploadStat) Check() {
+	s.mu.Lock()
+	s.Checked++
+	s.Refresh()
+	s.mu.Unlock()
+}
+
+func (s *FileToUploadStat) Add(filesize int64) {
+	s.mu.Lock()
+	s.Total++
+	s.TotalBytes += filesize
+	s.Progress.ChangeMax64(s.TotalBytes)
+	s.Refresh()
+	s.mu.Unlock()
+}
+
+func (s *FileToUploadStat) Commit(filereaded int64) {
+	s.mu.Lock()
+	s.UploadedBytes += filereaded
+	s.Progress.Set64(s.UploadedBytes)
+	s.mu.Unlock()
+}
+
+func (s *FileToUploadStat) Done() {
+	s.mu.Lock()
+	s.Uploaded++
+	s.Refresh()
+	s.mu.Unlock()
+}
+
+func (s *FileToUploadStat) Close() {
+	s.Progress.Close()
+}
+
+func (s *FileToUploadStat) Fail() {
+	s.mu.Lock()
+	s.Failed++
+	s.Refresh()
+	s.mu.Unlock()
+}
+
+func (s *FileToUploadStat) Skip() {
+	s.mu.Lock()
+	s.Skipped++
+	s.Refresh()
+	s.mu.Unlock()
+}

internal/piwigo/files.go

@@ -5,20 +5,13 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/url"
-	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 
-	"github.com/schollz/progressbar/v3"
 	"golang.org/x/text/unicode/norm"
 )
 
-type FileUploadResult struct {
-	ImageId int    `json:"image_id"`
-	Url     string `json:"url"`
-}
-
 func (p *Piwigo) FileExists(md5 string) bool {
 	var resp map[string]*string
 
@@ -31,35 +24,32 @@ func (p *Piwigo) FileExists(md5 string) bool {
 	return resp[md5] != nil
 }
 
-func (p *Piwigo) UploadChunks(filename string, nbJobs int, categoryId int) (*FileUploadResult, error) {
+func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, hasVideoJS bool) error {
-	md5, err := Md5File(filename, false)
+	if file.MD5() == "" {
-	if err != nil {
+		stat.Fail()
-		return nil, err
+		return errors.New("checksum error")
 	}
 
-	if p.FileExists(md5) {
+	stat.Check()
-		return nil, errors.New("file already exists")
+
+	if p.FileExists(file.MD5()) {
+		stat.Skip()
+		return errors.New("file already exists")
 	}
 
-	st, _ := os.Stat(filename)
 	wg := &sync.WaitGroup{}
-	chunks, err := Base64Chunker(filename)
+	chunks, err := Base64Chunker(file.FullPath())
 	errout := make(chan error)
-	bar := progressbar.DefaultBytes(
-		st.Size(),
-		"uploading",
-	)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	for j := 0; j < nbJobs; j++ {
 		wg.Add(1)
-		go p.UploadChunk(md5, chunks, errout, wg, bar)
+		go p.UploadChunk(file.MD5(), chunks, errout, wg, stat)
 	}
 	go func() {
 		wg.Wait()
-		bar.Close()
 		close(errout)
 	}()
 
@@ -68,30 +58,40 @@ func (p *Piwigo) UploadChunks(filename string, nbJobs int, categoryId int) (*Fil
 		errstring += err.Error() + "\n"
 	}
 	if errstring != "" {
-		return nil, errors.New(errstring)
+		stat.Fail()
+		return errors.New(errstring)
 	}
 
-	exif, _ := Exif(filename)
+	exif, _ := Exif(file.FullPath())
 	var resp *FileUploadResult
 	data := &url.Values{}
-	data.Set("original_sum", md5)
+	data.Set("original_sum", file.MD5())
-	data.Set("original_filename", filepath.Base(filename))
+	data.Set("original_filename", file.Name)
 	data.Set("check_uniqueness", "true")
 	if exif != nil && exif.CreatedAt != nil {
 		data.Set("date_creation", exif.CreatedAt.String())
 	}
-	if categoryId > 0 {
+	if file.CategoryId > 0 {
-		data.Set("categories", fmt.Sprint(categoryId))
+		data.Set("categories", fmt.Sprint(file.CategoryId))
 	}
 	err = p.Post("pwg.images.add", data, &resp)
 	if err != nil {
-		return nil, err
+		stat.Fail()
+		return err
 	}
 
-	return resp, nil
+	if hasVideoJS {
+		switch file.Ext() {
+		case "ogg", "ogv", "mp4", "m4v", "webm", "webmv":
+			p.VideoJSSync(resp.ImageId)
+		}
+	}
+
+	stat.Done()
+	return nil
 }
 
-func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, bar *progressbar.ProgressBar) {
+func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, progress *FileToUploadStat) {
 	defer wg.Done()
 	for chunk := range chunks {
 		var err error
@@ -107,7 +107,7 @@ func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout
 				break
 			}
 		}
-		bar.Add64(chunk.Size)
+		progress.Commit(chunk.Size)
 		if err != nil {
 			errout <- fmt.Errorf("error on chunk %d: %v", chunk.Position, err)
 			continue
@@ -115,12 +115,6 @@ func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout
 	}
 }
 
-type FileToUpload struct {
-	Filename   string
-	Md5        string
-	CategoryId int
-}
-
 func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, filter UploadFileType) ([]FileToUpload, error) {
 	rootPath, err := filepath.Abs(rootPath)
 	if err != nil {
@@ -147,7 +141,7 @@ func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, fi
 				continue
 			}
 			filename := filepath.Join(rootPath, dir.Name())
-			md5, err := Md5File(filename, false)
+			md5, err := Md5File(filename)
 			if err != nil {
 				return nil, err
 			}
@@ -158,8 +152,8 @@ func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, fi
 			fmt.Printf("%s - %s %s - %s\n", levelStr, dir.Name(), md5, status)
 			if status == "OK" {
 				files = append(files, FileToUpload{
-					Filename:   filename,
+					Dir:        rootPath,
-					Md5:        md5,
+					Name:       dir.Name(),
 					CategoryId: parentCategoryId,
 				})
 			}

internal/piwigo/helper.go

@@ -11,8 +11,6 @@ import (
 	"net/url"
 	"os"
 	"strings"
-
-	"github.com/schollz/progressbar/v3"
 )
 
 var CHUNK_BUFF_SIZE int64 = 32 * 1024
@@ -39,26 +37,20 @@ func ArgsToForm(args []string) (*url.Values, error) {
 	return params, nil
 }
 
-func Md5File(filename string, progress bool) (string, error) {
+func Md5File(filename string) (result string, err error) {
 	file, err := os.Open(filename)
 	if err != nil {
-		return "", err
+		return
 	}
 	defer file.Close()
 
-	st, _ := file.Stat()
 	hash := md5.New()
-
+	_, err = io.Copy(hash, file)
-	if progress {
-		bar := progressbar.DefaultBytes(st.Size(), "checksumming")
-		_, err = io.Copy(io.MultiWriter(hash, bar), file)
-	} else {
-		_, err = io.Copy(hash, file)
-	}
 	if err != nil {
-		return "", err
+		return
 	}
-	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+	result = fmt.Sprintf("%x", hash.Sum(nil))
+	return
 }
 
 type Base64ChunkResult struct {

internal/piwigocli/images_upload.go

@@ -2,11 +2,11 @@ package piwigocli
 
 import (
 	"errors"
-	"fmt"
 	"path/filepath"
 	"strings"
 
 	"github.com/celogeek/piwigo-cli/internal/piwigo"
+	"github.com/schollz/progressbar/v3"
 )
 
 type ImagesUploadCommand struct {
@@ -31,21 +31,25 @@ func (c *ImagesUploadCommand) Execute(args []string) error {
 		return errors.New("unsupported file extension")
 	}
 
-	resp, err := p.UploadChunks(c.Filename, c.NbJobs, c.CategoryId)
+	_, hasVideoJS := status.Plugins["piwigo-videojs"]
+
+	file := &piwigo.FileToUpload{
+		Dir:        filepath.Dir(c.Filename),
+		Name:       filepath.Base(c.Filename),
+		CategoryId: c.CategoryId,
+	}
+
+	stat := &piwigo.FileToUploadStat{
+		Progress: progressbar.DefaultBytes(1, "prepare"),
+	}
+	defer stat.Close()
+	stat.Add(file.Size())
+	stat.Refresh()
+
+	err = p.Upload(file, stat, c.NbJobs, hasVideoJS)
 	if err != nil {
 		return err
 	}
 
-	if _, ok := status.Plugins["piwigo-videojs"]; ok {
-		switch ext {
-		case "ogg", "ogv", "mp4", "m4v", "webm", "webmv":
-			fmt.Println("syncing metadata with videojs")
-			err = p.VideoJSSync(resp.ImageId)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
 	return nil
 }