From cabd7084fcb99f8511281c550f437725bd228640 Mon Sep 17 00:00:00 2001 From: celogeek <65178+celogeek@users.noreply.github.com> Date: Fri, 31 Dec 2021 16:26:31 +0100 Subject: [PATCH] merge file info --- internal/base64/chunker.go | 55 ------ internal/exif/extract.go | 56 ------ internal/md5/file.go | 22 --- internal/piwigo/files.go | 13 +- internal/piwigo/piwigotools/file_to_upload.go | 186 +++++++++++++++--- 5 files changed, 167 insertions(+), 165 deletions(-) delete mode 100644 internal/base64/chunker.go delete mode 100644 internal/exif/extract.go delete mode 100644 internal/md5/file.go diff --git a/internal/base64/chunker.go b/internal/base64/chunker.go deleted file mode 100644 index 5ab8616..0000000 --- a/internal/base64/chunker.go +++ /dev/null @@ -1,55 +0,0 @@ -package base64 - -import ( - "bytes" - b64 "encoding/base64" - "os" -) - -var CHUNK_SIZE int64 = 1 * 1024 * 1024 -var CHUNK_BUFF_SIZE int64 = 32 * 1024 -var CHUNK_BUFF_COUNT = CHUNK_SIZE / CHUNK_BUFF_SIZE - -type Chunk struct { - Position int64 - Size int64 - Buffer bytes.Buffer -} - -func Chunker(filename string) (chan *Chunk, error) { - f, err := os.Open(filename) - if err != nil { - return nil, err - } - - out := make(chan *Chunk, 8) - chunker := func() { - b := make([]byte, CHUNK_BUFF_SIZE) - defer f.Close() - defer close(out) - ok := false - for position := int64(0); !ok; position += 1 { - bf := &Chunk{ - Position: position, - } - b64 := b64.NewEncoder(b64.StdEncoding, &bf.Buffer) - for i := int64(0); i < CHUNK_BUFF_COUNT; i++ { - n, _ := f.Read(b) - if n == 0 { - ok = true - break - } - bf.Size += int64(n) - b64.Write(b[:n]) - } - b64.Close() - if bf.Size > 0 { - out <- bf - } - } - } - - go chunker() - - return out, nil -} diff --git a/internal/exif/extract.go b/internal/exif/extract.go deleted file mode 100644 index 3b2acac..0000000 --- a/internal/exif/extract.go +++ /dev/null @@ -1,56 +0,0 @@ -package exif - -import ( - "fmt" - "time" - - "github.com/barasher/go-exiftool" -) - -type Info struct { - CreatedAt *time.Time -} - -var ( - CreateDateFormat = "2006:01:02 15:04:05-07:00" -) - -func Extract(filename string) (*Info, error) { - et, err := exiftool.NewExiftool() - if err != nil { - return nil, err - } - defer et.Close() - - var resp *Info = &Info{} - fileInfos := et.ExtractMetadata(filename) - for _, fileInfo := range fileInfos { - if fileInfo.Err != nil { - continue - } - - var t time.Time - for k, v := range fileInfo.Fields { - switch k { - case "CreateDate": - offset, ok := fileInfo.Fields["OffsetTime"] - if !ok { - offset = "+00:00" - } - v := fmt.Sprintf("%s%s", v, offset) - t, err = time.Parse(CreateDateFormat, v) - case "CreationDate": - t, err = time.Parse(CreateDateFormat, fmt.Sprint(v)) - default: - continue - } - if err != nil { - continue - } - if resp.CreatedAt == nil || resp.CreatedAt.After(t) { - resp.CreatedAt = &t - } - } - } - return resp, nil -} diff --git a/internal/md5/file.go b/internal/md5/file.go deleted file mode 100644 index cd2ac3f..0000000 --- a/internal/md5/file.go +++ /dev/null @@ -1,22 +0,0 @@ -package md5 - -import ( - "crypto/md5" - "fmt" - "io" - "os" -) - -func File(filename string) (string, error) { - file, err := os.Open(filename) - if err != nil { - return "", err - } - defer file.Close() - - hash := md5.New() - if _, err = io.Copy(hash, file); err != nil { - return "", err - } - return fmt.Sprintf("%x", hash.Sum(nil)), nil -} diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go index 041bb2f..dea3f44 100644 --- a/internal/piwigo/files.go +++ b/internal/piwigo/files.go @@ -8,8 +8,6 @@ import ( "strings" "sync" - "github.com/celogeek/piwigo-cli/internal/base64" - "github.com/celogeek/piwigo-cli/internal/exif" "github.com/celogeek/piwigo-cli/internal/piwigo/piwigotools" "golang.org/x/text/unicode/norm" ) @@ -59,7 +57,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo return } wg := &sync.WaitGroup{} - chunks, err := base64.Chunker(file.FullPath()) + chunks, err := file.Base64Chunker() if err != nil { stat.Error("Base64Chunker", file.FullPath(), err) return @@ -82,10 +80,9 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo data.Set("original_sum", file.MD5()) data.Set("original_filename", file.Name) data.Set("check_uniqueness", "true") - - info, _ := exif.Extract(file.FullPath()) - if info != nil && info.CreatedAt != nil { - data.Set("date_creation", piwigotools.TimeResult(*info.CreatedAt).String()) + if file.CreatedAt() != nil { + fmt.Println(file.CreatedAt()) + data.Set("date_creation", file.CreatedAt().String()) } if file.CategoryId > 0 { data.Set("categories", fmt.Sprint(file.CategoryId)) @@ -117,7 +114,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo stat.Done() } -func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *base64.Chunk, wg *sync.WaitGroup, stat *piwigotools.FileToUploadStat, ok *bool) { +func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *piwigotools.FileToUploadChunk, wg *sync.WaitGroup, stat *piwigotools.FileToUploadStat, ok *bool) { defer wg.Done() for chunk := range chunks { var err error diff --git a/internal/piwigo/piwigotools/file_to_upload.go b/internal/piwigo/piwigotools/file_to_upload.go index 618256b..c063de0 100644 --- a/internal/piwigo/piwigotools/file_to_upload.go +++ b/internal/piwigo/piwigotools/file_to_upload.go @@ -1,58 +1,196 @@ package piwigotools import ( + "bytes" + "crypto/md5" + "encoding/base64" + "fmt" + "io" "os" "path/filepath" "strings" + "time" - "github.com/celogeek/piwigo-cli/internal/md5" + "github.com/barasher/go-exiftool" ) +type FileInfo struct { + md5 string + size int64 + ext string + createdAt *TimeResult +} + type FileToUpload struct { Dir string Name string CategoryId int - md5 *string - size *int64 - ext *string + info *FileInfo } func (f *FileToUpload) FullPath() string { return filepath.Join(f.Dir, f.Name) } +func (f *FileToUpload) Info() *FileInfo { + if f.info != nil { + return f.info + } + + file, err := os.Open(f.FullPath()) + if err != nil { + return nil + } + defer file.Close() + + st, err := file.Stat() + if err != nil { + return nil + } + + hash := md5.New() + if _, err = io.Copy(hash, file); err != nil { + return nil + } + + checksum := fmt.Sprintf("%x", hash.Sum(nil)) + + info := FileInfo{ + size: st.Size(), + ext: strings.ToLower(filepath.Ext(f.Name)[1:]), + md5: checksum, + createdAt: f.exifCreatedAt(), + } + + f.info = &info + return f.info +} + func (f *FileToUpload) Checked() bool { - return f.md5 != nil + return f.info != nil } func (f *FileToUpload) MD5() string { - if f.md5 == nil { - md5, err := md5.File(f.FullPath()) - if err != nil { - return "" - } - f.md5 = &md5 + if info := f.Info(); info != nil { + return info.md5 } - return *f.md5 + return "" } func (f *FileToUpload) Size() int64 { - if f.size == nil { - st, err := os.Stat(f.FullPath()) - if err != nil { - return -1 - } - size := st.Size() - f.size = &size + if info := f.Info(); info != nil { + return info.size } - return *f.size + return -1 } func (f *FileToUpload) Ext() string { - if f.ext == nil { - ext := strings.ToLower(filepath.Ext(f.Name)[1:]) - f.ext = &ext + if info := f.Info(); info != nil { + return info.ext } - return *f.ext + return "" +} + +func (f *FileToUpload) CreatedAt() *TimeResult { + if info := f.Info(); info != nil { + return info.createdAt + } + return nil +} + +var ( + CHUNK_SIZE int64 = 1 * 1024 * 1024 + CHUNK_BUFF_SIZE int64 = 32 * 1024 + CHUNK_BUFF_COUNT = CHUNK_SIZE / CHUNK_BUFF_SIZE +) + +type FileToUploadChunk struct { + Position int64 + Size int64 + Buffer bytes.Buffer +} + +func (f *FileToUpload) Base64Chunker() (chan *FileToUploadChunk, error) { + fh, err := os.Open(f.FullPath()) + if err != nil { + return nil, err + } + + out := make(chan *FileToUploadChunk, 8) + chunker := func() { + b := make([]byte, CHUNK_BUFF_SIZE) + defer fh.Close() + defer close(out) + ok := false + for position := int64(0); !ok; position += 1 { + bf := &FileToUploadChunk{ + Position: position, + } + b64 := base64.NewEncoder(base64.StdEncoding, &bf.Buffer) + for i := int64(0); i < CHUNK_BUFF_COUNT; i++ { + n, _ := fh.Read(b) + if n == 0 { + ok = true + break + } + bf.Size += int64(n) + b64.Write(b[:n]) + } + b64.Close() + if bf.Size > 0 { + out <- bf + } + } + } + + go chunker() + + return out, nil +} + +func (f *FileToUpload) exifCreatedAt() *TimeResult { + et, err := exiftool.NewExiftool() + if err != nil { + return nil + } + defer et.Close() + + var createdAt *time.Time + var CreateDateFormat = "2006:01:02 15:04:05-07:00" + + fileInfos := et.ExtractMetadata(f.FullPath()) + for _, fileInfo := range fileInfos { + if fileInfo.Err != nil { + continue + } + + var t time.Time + for k, v := range fileInfo.Fields { + switch k { + case "CreateDate": + offset, ok := fileInfo.Fields["OffsetTime"] + if !ok { + offset = "+00:00" + } + v := fmt.Sprintf("%s%s", v, offset) + t, err = time.Parse(CreateDateFormat, v) + case "CreationDate": + t, err = time.Parse(CreateDateFormat, fmt.Sprint(v)) + default: + continue + } + if err != nil { + continue + } + if createdAt == nil || createdAt.After(t) { + createdAt = &t + } + } + } + if createdAt != nil { + result := TimeResult(*createdAt) + return &result + } + return nil }