From f60d6f843fd3abe4f3fb38dccd2e918d10fbdd44 Mon Sep 17 00:00:00 2001
From: celogeek <65178+celogeek@users.noreply.github.com>
Date: Tue, 28 Dec 2021 12:14:04 +0100
Subject: [PATCH] add lock and retry

---
 internal/piwigo/files.go                 | 39 +++++++++++++++++++++---
 internal/piwigo/piwigo.go                |  4 +++
 internal/piwigocli/images_upload_tree.go |  2 +-
 3 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go
index d9992fd..d7c8138 100644
--- a/internal/piwigo/files.go
+++ b/internal/piwigo/files.go
@@ -68,6 +68,8 @@ func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int,
 		return
 	}
 
+	// lock this process for committing the file
+
 	exif, _ := Exif(file.FullPath())
 	var resp *FileUploadResult
 	data := &url.Values{}
@@ -80,10 +82,20 @@ func (p *Piwigo) Upload(file *FileToUpload, stat *FileToUploadStat, nbJobs int,
 	if file.CategoryId > 0 {
 		data.Set("categories", fmt.Sprint(file.CategoryId))
 	}
-	err = p.Post("pwg.images.add", data, &resp)
+
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	for i := 0; i < 3; i++ {
+		err = p.Post("pwg.images.add", data, &resp)
+		if err == nil || err.Error() == "[Error 500] file already exists" {
+			err = nil
+			break
+		}
+		stat.Error(fmt.Sprintf("Upload %d", i), file.FullPath(), err)
+	}
+
 	if err != nil {
 		stat.Fail()
-		stat.Error("Upload", file.FullPath(), err)
 		return
 	}
 
@@ -161,10 +173,12 @@ func (p *Piwigo) ScanTree(
 				var resp struct {
 					Id int `json:"id"`
 				}
+				p.mu.Lock()
 				err = p.Post("pwg.categories.add", &url.Values{
 					"name":   []string{strings.ReplaceAll(dirname, "'", `\'`)},
 					"parent": []string{fmt.Sprint(parentCategoryId)},
 				}, &resp)
+				p.mu.Unlock()
 				if err != nil {
 					stat.Error("ScanTree Categories Add", rootPath, err)
 					return
@@ -209,9 +223,24 @@ func (p *Piwigo) CheckFiles(filesToCheck chan *FileToUpload, files chan *FileToU
 	wg.Wait()
 }
 
-func (p *Piwigo) UploadFiles(files chan *FileToUpload, stat *FileToUploadStat, hasVideoJS bool, nbJobs int) {
+func (p *Piwigo) UploadFiles(
+	files chan *FileToUpload,
+	stat *FileToUploadStat,
+	hasVideoJS bool,
+	nbJobs int,
+	nbJobsChunk int,
+) {
 	defer stat.Close()
-	for file := range files {
-		p.Upload(file, stat, nbJobs, hasVideoJS)
+
+	wg := &sync.WaitGroup{}
+	for i := 0; i < nbJobs; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for file := range files {
+				p.Upload(file, stat, nbJobsChunk, hasVideoJS)
+			}
+		}()
 	}
+	wg.Wait()
 }
diff --git a/internal/piwigo/piwigo.go b/internal/piwigo/piwigo.go
index b4e353a..7bb4e31 100644
--- a/internal/piwigo/piwigo.go
+++ b/internal/piwigo/piwigo.go
@@ -1,10 +1,14 @@
 package piwigo
 
+import "sync"
+
 type Piwigo struct {
 	Url      string `json:"url"`
 	Username string `json:"username"`
 	Password string `json:"password"`
 	Token    string `json:"token"`
+
+	mu sync.Mutex
 }
 
 type PiwigoResult struct {
diff --git a/internal/piwigocli/images_upload_tree.go b/internal/piwigocli/images_upload_tree.go
index 6dac6e6..16afacd 100644
--- a/internal/piwigocli/images_upload_tree.go
+++ b/internal/piwigocli/images_upload_tree.go
@@ -33,7 +33,7 @@ func (c *ImagesUploadTreeCommand) Execute(args []string) error {
 
 	go p.ScanTree(c.Dirname, c.CategoryId, 0, &status.UploadFileType, stat, filesToCheck)
 	go p.CheckFiles(filesToCheck, files, stat, 8)
-	p.UploadFiles(files, stat, hasVideoJS, 4)
+	p.UploadFiles(files, stat, hasVideoJS, 4, 2)
 
 	return nil
 }