From b6917ae0e1885ace9eb54143bb638c602f013763 Mon Sep 17 00:00:00 2001 From: celogeek <65178+celogeek@users.noreply.github.com> Date: Mon, 27 Dec 2021 20:13:06 +0100 Subject: [PATCH] improve error display --- internal/piwigo/file_to_upload_stat.go | 9 +++ internal/piwigo/files.go | 96 ++++++++++-------------- internal/piwigo/helper.go | 4 +- internal/piwigocli/images_upload.go | 5 +- internal/piwigocli/images_upload_tree.go | 8 +- 5 files changed, 56 insertions(+), 66 deletions(-) diff --git a/internal/piwigo/file_to_upload_stat.go b/internal/piwigo/file_to_upload_stat.go index cf5ef1f..b27fe43 100644 --- a/internal/piwigo/file_to_upload_stat.go +++ b/internal/piwigo/file_to_upload_stat.go @@ -76,3 +76,12 @@ func (s *FileToUploadStat) Skip() { s.Refresh() s.mu.Unlock() } + +func (s *FileToUploadStat) Error(origin string, err error) error { + s.mu.Lock() + s.Progress.Clear() + fmt.Println("%s: %s", origin, err) + s.Progress.RenderBlank() + s.mu.Unlock() + return err +} diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go index 15f69b0..d3312ff 100644 --- a/internal/piwigo/files.go +++ b/internal/piwigo/files.go @@ -1,7 +1,6 @@ package piwigo import ( - "errors" "fmt" "io/ioutil" "net/url" @@ -24,18 +23,21 @@ func (p *Piwigo) FileExists(md5 string) bool { return resp[md5] != nil } -func (p *Piwigo) CheckUploadFile(file *FileToUpload, stat *FileToUploadStat) error { +func (p *Piwigo) CheckUploadFile(file *FileToUpload, stat *FileToUploadStat) (err error) { if !file.Checked() { if file.MD5() == "" { stat.Fail() stat.Check() - return errors.New("checksum error") + err = fmt.Errorf("%s: checksum error", file.FullPath()) + stat.Error(file.FullPath(), err) + return } if p.FileExists(file.MD5()) { stat.Skip() stat.Check() - return errors.New("file already exists") + err = fmt.Errorf("%s: file already exists", file.FullPath()) + return } stat.Check() @@ -44,34 +46,26 @@ func (p *Piwigo) CheckUploadFile(file *FileToUpload, stat *FileToUploadStat) err return nil } -func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, hasVideoJS bool) error { +func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, hasVideoJS bool) { err := p.CheckUploadFile(file, stat) if err != nil { - return err + return } wg := &sync.WaitGroup{} chunks, err := Base64Chunker(file.FullPath()) - errout := make(chan error) if err != nil { - return err + stat.Error(file.FullPath(), err) + return } + ok := true for j := 0; j < nbJobs; j++ { wg.Add(1) - go p.UploadChunk(file.MD5(), chunks, errout, wg, stat) + go p.UploadChunk(file, chunks, wg, stat, &ok) } - go func() { - wg.Wait() - close(errout) - }() - - var errstring string - for err := range errout { - errstring += err.Error() + "\n" - } - if errstring != "" { - stat.Fail() - return errors.New(errstring) + wg.Wait() + if !ok { + return } exif, _ := Exif(file.FullPath()) @@ -89,7 +83,8 @@ func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, err = p.Post("pwg.images.add", data, &resp) if err != nil { stat.Fail() - return err + stat.Error(file.FullPath(), err) + return } if hasVideoJS { @@ -100,15 +95,14 @@ func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, } stat.Done() - return nil } -func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, progress *FileToUploadStat) { +func (p *Piwigo) UploadChunk(file *FileToUpload, chunks chan *Base64ChunkResult, wg *sync.WaitGroup, stat *FileToUploadStat, ok *bool) { defer wg.Done() for chunk := range chunks { var err error data := &url.Values{ - "original_sum": []string{md5}, + "original_sum": []string{file.MD5()}, "position": []string{fmt.Sprint(chunk.Position)}, "type": []string{"file"}, "data": []string{chunk.Buffer.String()}, @@ -119,10 +113,12 @@ func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout break } } - progress.Commit(chunk.Size) + stat.Commit(chunk.Size) if err != nil { - errout <- fmt.Errorf("error on chunk %d: %v", chunk.Position, err) - continue + stat.Fail() + stat.Error(file.FullPath(), err) + *ok = false + return } } } @@ -134,22 +130,25 @@ func (p *Piwigo) ScanTree( filter *UploadFileType, stat *FileToUploadStat, files chan *FileToUpload, -) (err error) { +) { if level == 0 { defer close(files) } - rootPath, err = filepath.Abs(rootPath) + rootPath, err := filepath.Abs(rootPath) if err != nil { + stat.Error(rootPath, err) return } categoriesId, err := p.CategoriesId(parentCategoryId) if err != nil { + stat.Error(rootPath, err) return } dirs, err := ioutil.ReadDir(rootPath) if err != nil { + stat.Error(rootPath, err) return } @@ -167,14 +166,12 @@ func (p *Piwigo) ScanTree( "parent": []string{fmt.Sprint(parentCategoryId)}, }, &resp) if err != nil { + stat.Error(rootPath, err) return } categoryId = resp.Id } - err = p.ScanTree(filepath.Join(rootPath, dirname), categoryId, level+1, filter, stat, files) - if err != nil { - return - } + p.ScanTree(filepath.Join(rootPath, dirname), categoryId, level+1, filter, stat, files) case false: // File file := &FileToUpload{ Dir: rootPath, @@ -189,7 +186,6 @@ func (p *Piwigo) ScanTree( } } - return nil } func (p *Piwigo) CheckFiles(filesToCheck chan *FileToUpload, files chan *FileToUpload, stat *FileToUploadStat, nbJobs int) { @@ -202,9 +198,10 @@ func (p *Piwigo) CheckFiles(filesToCheck chan *FileToUpload, files chan *FileToU defer wg.Done() for file := range filesToCheck { err := p.CheckUploadFile(file, stat) - if err == nil { - files <- file + if err != nil { + continue } + files <- file } }() } @@ -212,35 +209,18 @@ func (p *Piwigo) CheckFiles(filesToCheck chan *FileToUpload, files chan *FileToU wg.Wait() } -func (p *Piwigo) UploadFiles(files chan *FileToUpload, stat *FileToUploadStat, hasVideoJS bool, nbJobs int) error { +func (p *Piwigo) UploadFiles(files chan *FileToUpload, stat *FileToUploadStat, hasVideoJS bool, nbJobs int) { defer stat.Close() - errchan := make(chan error) - wg := &sync.WaitGroup{} + wg := &sync.WaitGroup{} for i := 0; i < nbJobs; i++ { wg.Add(1) go func() { defer wg.Done() for file := range files { - err := p.Upload(file, stat, 1, hasVideoJS) - if err != nil { - errchan <- fmt.Errorf("%s: %s", file.FullPath(), err.Error()) - } + p.Upload(file, stat, 2, hasVideoJS) } }() } - go func() { - wg.Wait() - close(errchan) - }() - - errstring := "" - for err := range errchan { - errstring += err.Error() - } - if errstring != "" { - return errors.New(errstring) - } - - return nil + wg.Wait() } diff --git a/internal/piwigo/helper.go b/internal/piwigo/helper.go index 8041f1a..976bc9e 100644 --- a/internal/piwigo/helper.go +++ b/internal/piwigo/helper.go @@ -86,7 +86,9 @@ func Base64Chunker(filename string) (out chan *Base64ChunkResult, err error) { b64.Write(b[:n]) } b64.Close() - out <- bf + if bf.Size > 0 { + out <- bf + } } }() diff --git a/internal/piwigocli/images_upload.go b/internal/piwigocli/images_upload.go index 5f9ac73..ab4f322 100644 --- a/internal/piwigocli/images_upload.go +++ b/internal/piwigocli/images_upload.go @@ -44,10 +44,7 @@ func (c *ImagesUploadCommand) Execute(args []string) error { } defer stat.Close() stat.Add() - err = p.Upload(file, stat, c.NbJobs, hasVideoJS) - if err != nil { - return err - } + p.Upload(file, stat, c.NbJobs, hasVideoJS) return nil } diff --git a/internal/piwigocli/images_upload_tree.go b/internal/piwigocli/images_upload_tree.go index e9488fc..59cddb2 100644 --- a/internal/piwigocli/images_upload_tree.go +++ b/internal/piwigocli/images_upload_tree.go @@ -26,12 +26,14 @@ func (c *ImagesUploadTreeCommand) Execute(args []string) error { stat := &piwigo.FileToUploadStat{ Progress: progressbar.DefaultBytes(1, "..."), } - defer stat.Close() + defer stat.Close() filesToCheck := make(chan *piwigo.FileToUpload, 1000) files := make(chan *piwigo.FileToUpload, 1000) go p.ScanTree(c.Dirname, c.CategoryId, 0, &status.UploadFileType, stat, filesToCheck) - go p.CheckFiles(filesToCheck, files, stat, 8) - return p.UploadFiles(files, stat, hasVideoJS, 8) + go p.CheckFiles(filesToCheck, files, stat, 2) + p.UploadFiles(files, stat, hasVideoJS, 4) + + return nil }