From 131f7038a9adc9bb847dd064cacd66ababda3af0 Mon Sep 17 00:00:00 2001 From: celogeek <65178+celogeek@users.noreply.github.com> Date: Mon, 27 Dec 2021 16:45:34 +0100 Subject: [PATCH] parallel checks --- internal/piwigo/file_to_upload_stat.go | 12 +- internal/piwigo/files.go | 137 ++++++++++++++--------- internal/piwigocli/images_upload.go | 6 +- internal/piwigocli/images_upload_tree.go | 18 ++- 4 files changed, 105 insertions(+), 68 deletions(-) diff --git a/internal/piwigo/file_to_upload_stat.go b/internal/piwigo/file_to_upload_stat.go index 2219702..430d010 100644 --- a/internal/piwigo/file_to_upload_stat.go +++ b/internal/piwigo/file_to_upload_stat.go @@ -30,15 +30,21 @@ func (s *FileToUploadStat) Check() { s.mu.Unlock() } -func (s *FileToUploadStat) Add(filesize int64) { +func (s *FileToUploadStat) AddBytes(filesize int64) { s.mu.Lock() - s.Total++ s.TotalBytes += filesize s.Progress.ChangeMax64(s.TotalBytes) s.Refresh() s.mu.Unlock() } +func (s *FileToUploadStat) Add() { + s.mu.Lock() + s.Total++ + s.Refresh() + s.mu.Unlock() +} + func (s *FileToUploadStat) Commit(filereaded int64) { s.mu.Lock() s.UploadedBytes += filereaded @@ -60,6 +66,7 @@ func (s *FileToUploadStat) Close() { func (s *FileToUploadStat) Fail() { s.mu.Lock() s.Failed++ + s.Checked++ s.Refresh() s.mu.Unlock() } @@ -67,6 +74,7 @@ func (s *FileToUploadStat) Fail() { func (s *FileToUploadStat) Skip() { s.mu.Lock() s.Skipped++ + s.Checked++ s.Refresh() s.mu.Unlock() } diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go index 9a8fae8..eb31823 100644 --- a/internal/piwigo/files.go +++ b/internal/piwigo/files.go @@ -24,20 +24,29 @@ func (p *Piwigo) FileExists(md5 string) bool { return resp[md5] != nil } -func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, hasVideoJS bool) error { +func (p *Piwigo) CheckUploadFile(file *FileToUpload, stat *FileToUploadStat) error { if !file.Checked() { if file.MD5() == "" { stat.Fail() return errors.New("checksum error") } + + if p.FileExists(file.MD5()) { + stat.Skip() + return errors.New("file already exists") + } + stat.Check() + stat.AddBytes(file.Size()) } + return nil +} - if p.FileExists(file.MD5()) { - stat.Skip() - return errors.New("file already exists") +func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int, hasVideoJS bool) error { + err := p.CheckUploadFile(file, stat) + if err != nil { + return err } - wg := &sync.WaitGroup{} chunks, err := Base64Chunker(file.FullPath()) errout := make(chan error) @@ -116,72 +125,88 @@ func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout } } -func (p *Piwigo) UploadTree(rootPath string, parentCategoryId int, level int, filter UploadFileType) ([]FileToUpload, error) { - rootPath, err := filepath.Abs(rootPath) +func (p *Piwigo) ScanTree( + rootPath string, + parentCategoryId int, + level int, + filter *UploadFileType, + stat *FileToUploadStat, + files chan *FileToUpload, +) (err error) { + if level == 0 { + defer close(files) + } + rootPath, err = filepath.Abs(rootPath) if err != nil { - return nil, err + return } categoriesId, err := p.CategoriesId(parentCategoryId) if err != nil { - return nil, err + return } dirs, err := ioutil.ReadDir(rootPath) if err != nil { - return nil, err + return } - var files []FileToUpload - - levelStr := strings.Repeat(" ", level) for _, dir := range dirs { - if !dir.IsDir() { - ext := strings.ToLower(filepath.Ext(dir.Name())[1:]) - if !filter.Has(ext) { + switch dir.IsDir() { + case true: // Directory + dirname := norm.NFC.String(dir.Name()) + categoryId, ok := categoriesId[dirname] + if !ok { + var resp struct { + Id int `json:"id"` + } + err = p.Post("pwg.categories.add", &url.Values{ + "name": []string{strings.ReplaceAll(dirname, "'", `\'`)}, + "parent": []string{fmt.Sprint(parentCategoryId)}, + }, &resp) + if err != nil { + return + } + categoryId = resp.Id + } + err = p.ScanTree(filepath.Join(rootPath, dirname), categoryId, level+1, filter, stat, files) + if err != nil { + return + } + case false: // File + file := &FileToUpload{ + Dir: rootPath, + Name: dir.Name(), + CategoryId: parentCategoryId, + } + if !filter.Has(file.Ext()) { continue } - filename := filepath.Join(rootPath, dir.Name()) - md5, err := Md5File(filename) - if err != nil { - return nil, err - } - status := "OK" - if p.FileExists(md5) { - status = "SKIP" - } - fmt.Printf("%s - %s %s - %s\n", levelStr, dir.Name(), md5, status) - if status == "OK" { - files = append(files, FileToUpload{ - Dir: rootPath, - Name: dir.Name(), - CategoryId: parentCategoryId, - }) - } - continue + stat.Add() + files <- file } - dirname := norm.NFC.String(dir.Name()) - categoryId, ok := categoriesId[dirname] - fmt.Printf("%s%s\n", levelStr, dirname) - if !ok { - var resp struct { - Id int `json:"id"` - } - err = p.Post("pwg.categories.add", &url.Values{ - "name": []string{strings.ReplaceAll(dirname, "'", `\'`)}, - "parent": []string{fmt.Sprint(parentCategoryId)}, - }, &resp) - if err != nil { - return nil, err - } - categoryId = resp.Id - } - newFiles, err := p.UploadTree(filepath.Join(rootPath, dirname), categoryId, level+1, filter) - if err != nil { - return nil, err - } - files = append(files, newFiles...) } - return files, nil + return nil +} + +func (p *Piwigo) CheckFiles(filesToCheck chan *FileToUpload, files chan *FileToUpload, stat *FileToUploadStat, nbJobs int) { + defer close(files) + + wgChecker := &sync.WaitGroup{} + for i := 0; i < nbJobs; i++ { + wgChecker.Add(1) + go func() { + defer wgChecker.Done() + for file := range filesToCheck { + err := p.CheckUploadFile(file, stat) + if err != nil { + continue + } + files <- file + } + }() + } + + wgChecker.Wait() } diff --git a/internal/piwigocli/images_upload.go b/internal/piwigocli/images_upload.go index 85fb026..5f9ac73 100644 --- a/internal/piwigocli/images_upload.go +++ b/internal/piwigocli/images_upload.go @@ -40,12 +40,10 @@ func (c *ImagesUploadCommand) Execute(args []string) error { } stat := &piwigo.FileToUploadStat{ - Progress: progressbar.DefaultBytes(1, "prepare"), + Progress: progressbar.DefaultBytes(1, "..."), } defer stat.Close() - stat.Add(file.Size()) - stat.Refresh() - + stat.Add() err = p.Upload(file, stat, c.NbJobs, hasVideoJS) if err != nil { return err diff --git a/internal/piwigocli/images_upload_tree.go b/internal/piwigocli/images_upload_tree.go index 247a90b..ef4fcf7 100644 --- a/internal/piwigocli/images_upload_tree.go +++ b/internal/piwigocli/images_upload_tree.go @@ -1,9 +1,8 @@ package piwigocli import ( - "fmt" - "github.com/celogeek/piwigo-cli/internal/piwigo" + "github.com/schollz/progressbar/v3" ) type ImagesUploadTreeCommand struct { @@ -23,11 +22,18 @@ func (c *ImagesUploadTreeCommand) Execute(args []string) error { return err } - files, err := p.UploadTree(c.Dirname, c.CategoryId, 0, status.UploadFileType) - if err != nil { - return err + stat := &piwigo.FileToUploadStat{ + Progress: progressbar.DefaultBytes(1, "..."), + } + defer stat.Close() + + filesToCheck := make(chan *piwigo.FileToUpload, 1000) + files := make(chan *piwigo.FileToUpload, 1000) + + go p.ScanTree(c.Dirname, c.CategoryId, 0, &status.UploadFileType, stat, filesToCheck) + go p.CheckFiles(filesToCheck, files, stat, 8) + for range files { } - fmt.Println("Total", len(files)) return nil }