upload file, ext check

internal/piwigo/files.go

@@ -10,6 +10,11 @@ import (
 	"github.com/schollz/progressbar/v3"
 )
 
+type FileUploadResult struct {
+	ImageId int    `json:"image_id"`
+	Url     string `json:"url"`
+}
+
 func (p *Piwigo) FileExists(md5 string) bool {
 	var resp map[string]*string
 
@@ -22,83 +27,79 @@ func (p *Piwigo) FileExists(md5 string) bool {
 	return resp[md5] != nil
 }
 
-func (p *Piwigo) UploadChunks(filename string, nbJobs int) error {
+func (p *Piwigo) UploadChunks(filename string, nbJobs int) (*FileUploadResult, error) {
 	md5, err := Md5File(filename)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	if p.FileExists(md5) {
-		return errors.New("file already exists")
+		return nil, errors.New("file already exists")
 	}
+	st, _ := os.Stat(filename)
 
-	f, err := os.Open(filename)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	st, err := f.Stat()
-	if err != nil {
-		return err
-	}
-
-	in := make(chan int64)
-	out := make(chan error)
 	wg := &sync.WaitGroup{}
+	chunks, err := Base64Chunker(filename)
+	errout := make(chan error)
 	bar := progressbar.DefaultBytes(
 		st.Size(),
 		"uploading",
 	)
+	if err != nil {
+		return nil, err
+	}
 
 	for j := 0; j < nbJobs; j++ {
 		wg.Add(1)
-		go p.UploadChunk(md5, f, in, out, wg, bar)
+		go p.UploadChunk(md5, chunks, errout, wg, bar)
 	}
-
 	go func() {
-		nbChunks := st.Size()/CHUNK_SIZE + 1
-		for position := int64(0); position < nbChunks; position++ {
-			in <- position
-		}
-		close(in)
 		wg.Wait()
-		close(out)
 		bar.Close()
+		close(errout)
 	}()
 
-	var errString string
+	var errstring string
-	for err := range out {
+	for err := range errout {
-		errString += err.Error() + "\n"
+		errstring += err.Error() + "\n"
 	}
-	if errString != "" {
+	if errstring != "" {
-		return errors.New(errString[:len(errString)-1])
+		return nil, errors.New(errstring)
 	}
 
-	fmt.Println(md5)
+	var resp *FileUploadResult
-
+	err = p.Post("pwg.images.add", &url.Values{
-	return nil
-}
-
-func (p *Piwigo) UploadChunk(md5 string, f *os.File, in chan int64, out chan error, wg *sync.WaitGroup, bar *progressbar.ProgressBar) {
-	defer wg.Done()
-	for position := range in {
-		n, b64, err := Base64Chunk(f, position)
-		if err != nil {
-			out <- fmt.Errorf("error on chunk %d: %v", position, err)
-			continue
-		}
-
-		err = p.Post("pwg.images.addChunk", &url.Values{
 		"original_sum":      []string{md5},
-			"position":     []string{fmt.Sprint(position)},
+		"original_filename": []string{filename},
-			"type":         []string{"file"},
+		"check_uniqueness":  []string{"true"},
-			"data":         []string{b64},
+	}, &resp)
-		}, nil)
 	if err != nil {
-			out <- fmt.Errorf("error on chunk %d: %v", position, err)
+		return nil, err
+	}
+
+	return resp, nil
+}
+
+func (p *Piwigo) UploadChunk(md5 string, chunks chan *Base64ChunkResult, errout chan error, wg *sync.WaitGroup, bar *progressbar.ProgressBar) {
+	defer wg.Done()
+	for chunk := range chunks {
+		var err error
+		data := &url.Values{
+			"original_sum": []string{md5},
+			"position":     []string{fmt.Sprint(chunk.Position)},
+			"type":         []string{"file"},
+			"data":         []string{chunk.Buffer.String()},
+		}
+		for i := 0; i < 3; i++ {
+			err = p.Post("pwg.images.addChunk", data, nil)
+			if err == nil {
+				break
+			}
+		}
+		bar.Add64(chunk.Size)
+		if err != nil {
+			errout <- fmt.Errorf("error on chunk %d: %v", chunk.Position, err)
 			continue
 		}
-		bar.Add(n)
 	}
 }

internal/piwigo/helper.go

@@ -1,19 +1,23 @@
 package piwigo
 
 import (
+	"bytes"
 	"crypto/md5"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"net/url"
 	"os"
 	"strings"
+
+	"github.com/schollz/progressbar/v3"
 )
 
-var CHUNK_SIZE int64 = int64(math.Pow(1024, 2))
+var CHUNK_BUFF_SIZE int64 = 32 * 1024
+var CHUNK_BUFF_COUNT int = 32
+var CHUNK_PRECOMPUTE_SIZE int = 8
 
 func DumpResponse(v interface{}) (err error) {
 	b, err := json.MarshalIndent(v, "", "  ")
@@ -40,22 +44,55 @@ func Md5File(filename string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	defer file.Close()
+
+	st, _ := file.Stat()
+	bar := progressbar.DefaultBytes(st.Size(), "checksumming")
+
 	hash := md5.New()
-	_, err = io.Copy(hash, file)
+	_, err = io.Copy(io.MultiWriter(hash, bar), file)
 	if err != nil {
 		return "", err
 	}
 	return fmt.Sprintf("%x", hash.Sum(nil)), nil
 }
 
-func Base64Chunk(file *os.File, position int64) (int, string, error) {
+type Base64ChunkResult struct {
-	b := make([]byte, CHUNK_SIZE)
+	Position int64
-	n, err := file.ReadAt(b, position*CHUNK_SIZE)
+	Size     int64
-	if err != nil && err != io.EOF {
+	Buffer   bytes.Buffer
-		return 0, "", err
+}
-	}
+
-	if n == 0 {
+func Base64Chunker(filename string) (out chan *Base64ChunkResult, err error) {
-		return 0, "", errors.New("position out of bound")
+	f, err := os.Open(filename)
-	}
+	if err != nil {
-	return n, base64.StdEncoding.EncodeToString(b[:n]), nil
+		return
+	}
+
+	out = make(chan *Base64ChunkResult, CHUNK_PRECOMPUTE_SIZE)
+	go func() {
+		b := make([]byte, CHUNK_BUFF_SIZE)
+		defer f.Close()
+		defer close(out)
+		ok := false
+		for position := int64(0); !ok; position += 1 {
+			bf := &Base64ChunkResult{
+				Position: position,
+			}
+			b64 := base64.NewEncoder(base64.StdEncoding, &bf.Buffer)
+			for i := 0; i < CHUNK_BUFF_COUNT; i++ {
+				n, _ := f.Read(b)
+				if n == 0 {
+					ok = true
+					break
+				}
+				bf.Size += int64(n)
+				b64.Write(b[:n])
+			}
+			b64.Close()
+			out <- bf
+		}
+	}()
+
+	return
 }

internal/piwigo/login.go

@@ -1,14 +1,44 @@
 package piwigo
 
 import (
+	"encoding/json"
 	"errors"
 	"net/url"
+	"strings"
 )
 
+type UploadFileType map[string]bool
+
 type StatusResponse struct {
 	User           string         `json:"username"`
 	Role           string         `json:"status"`
 	Version        string         `json:"version"`
+	Token          string         `json:"pwg_token"`
+	UploadFileType UploadFileType `json:"upload_file_types"`
+}
+
+func (uft *UploadFileType) UnmarshalJSON(data []byte) error {
+	var r string
+	if err := json.Unmarshal(data, &r); err != nil {
+		return err
+	}
+	*uft = UploadFileType{}
+	for _, v := range strings.Split(r, ",") {
+		(*uft)[v] = true
+	}
+	return nil
+}
+
+func (uft UploadFileType) MarshalJSON() ([]byte, error) {
+	return []byte(`"` + uft.String() + `"`), nil
+}
+
+func (uft UploadFileType) String() string {
+	keys := make([]string, 0, len(uft))
+	for k, _ := range uft {
+		keys = append(keys, k)
+	}
+	return strings.Join(keys, ",")
 }
 
 func (p *Piwigo) GetStatus() (*StatusResponse, error) {
@@ -22,7 +52,6 @@ func (p *Piwigo) GetStatus() (*StatusResponse, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	if resp.User == p.Username {
 		return resp, nil
 	}

internal/piwigo/post.go

@@ -61,7 +61,7 @@ func (p *Piwigo) Post(method string, form *url.Values, resp interface{}) error {
 		newBody := &bytes.Buffer{}
 		tee := io.TeeReader(r.Body, newBody)
 
-		var RawResult map[string]interface{}
+		var RawResult interface{}
 		err = json.NewDecoder(tee).Decode(&RawResult)
 		if err != nil {
 			return err

internal/piwigo/videojs.go

@@ -0,0 +1,36 @@
+package piwigo
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+)
+
+func (p *Piwigo) VideoJSSync(imageId int) error {
+	Url, err := url.Parse(p.Url)
+	if err != nil {
+		return err
+	}
+	Url.Path += "/admin.php"
+	q := Url.Query()
+	q.Set("page", "plugin")
+	q.Set("section", "piwigo-videojs/admin/admin_photo.php")
+	q.Set("sync_metadata", "1")
+	q.Set("image_id", fmt.Sprint(imageId))
+	Url.RawQuery = q.Encode()
+
+	req, err := http.NewRequest("GET", Url.String(), nil)
+	if err != nil {
+		return err
+	}
+	if p.Token != "" {
+		req.AddCookie(&http.Cookie{Name: "pwg_id", Value: p.Token, HttpOnly: true})
+	}
+
+	r, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer r.Body.Close()
+	return nil
+}

internal/piwigocli/images_upload.go

@@ -1,6 +1,10 @@
 package piwigocli
 
 import (
+	"errors"
+	"path/filepath"
+	"strings"
+
 	"github.com/celogeek/piwigo-cli/internal/piwigo"
 )
 
@@ -15,15 +19,28 @@ func (c *ImagesUploadCommand) Execute(args []string) error {
 		return err
 	}
 
-	_, err := p.Login()
+	status, err := p.Login()
 	if err != nil {
 		return err
 	}
 
-	err = p.UploadChunks(c.Filename, c.NBJobs)
+	ext := strings.ToLower(filepath.Ext(c.Filename)[1:])
+	if _, ok := status.UploadFileType[ext]; !ok {
+		return errors.New("unsupported file extension")
+	}
+
+	resp, err := p.UploadChunks(c.Filename, c.NBJobs)
 	if err != nil {
 		return err
 	}
 
+	switch ext {
+	case "ogg", "ogv", "mp4", "m4v", "webm", "webmv":
+		err = p.VideoJSSync(resp.ImageId)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

internal/piwigocli/session_status.go

@@ -28,6 +28,8 @@ func (c *SessionStatusCommand) Execute(args []string) error {
 		{"Version", resp.Version},
 		{"User", resp.User},
 		{"Role", resp.Role},
+		{"Admin Token", resp.Token},
+		{"Supported formats", resp.UploadFileType},
 	})
 
 	t.SetOutputMirror(os.Stdout)