From 234a63f208920f8e5ae0306060aa6cccab911c49 Mon Sep 17 00:00:00 2001 From: celogeek <65178+celogeek@users.noreply.github.com> Date: Sun, 2 Jan 2022 16:39:49 +0100 Subject: [PATCH] lazy compute file --- internal/piwigo/files.go | 53 ++--- internal/piwigo/piwigotools/file_to_upload.go | 195 +++++++++--------- 2 files changed, 126 insertions(+), 122 deletions(-) diff --git a/internal/piwigo/files.go b/internal/piwigo/files.go index ea82a61..8f28e0d 100644 --- a/internal/piwigo/files.go +++ b/internal/piwigo/files.go @@ -29,25 +29,28 @@ func (p *Piwigo) FileExists(md5 string) bool { } func (p *Piwigo) CheckUploadFile(file *piwigotools.FileToUpload, stat *piwigotools.FileToUploadStat) (err error) { - if !file.Checked() { - if file.MD5() == "" { - stat.Fail() - stat.Check() - err = fmt.Errorf("%s: checksum error", file.FullPath()) - stat.Error("CheckUploadFile", file.FullPath(), err) - return - } - - if p.FileExists(file.MD5()) { - stat.Skip() - stat.Check() - err = fmt.Errorf("%s: file already exists", file.FullPath()) - return - } - - stat.Check() - stat.AddBytes(file.Size()) + if file.Checked() { + return nil } + + if file.MD5() == nil { + stat.Fail() + stat.Check() + err = fmt.Errorf("%s: checksum error", *file.FullPath()) + stat.Error("CheckUploadFile", *file.FullPath(), err) + return + } + + if p.FileExists(*file.MD5()) { + stat.Skip() + stat.Check() + err = fmt.Errorf("%s: file already exists", *file.FullPath()) + return + } + + stat.Check() + stat.AddBytes(*file.Size()) + return nil } @@ -59,7 +62,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo wg := &sync.WaitGroup{} chunks, err := file.Base64Chunker() if err != nil { - stat.Error("Base64Chunker", file.FullPath(), err) + stat.Error("Base64Chunker", *file.FullPath(), err) return } @@ -77,7 +80,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo var resp *FileUploadResult data := &url.Values{} - data.Set("original_sum", file.MD5()) + data.Set("original_sum", *file.MD5()) data.Set("original_filename", file.Name) data.Set("check_uniqueness", "true") if file.CreatedAt() != nil { @@ -95,7 +98,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo err = nil break } - stat.Error(fmt.Sprintf("Upload %d", i), file.FullPath(), err) + stat.Error(fmt.Sprintf("Upload %d", i), *file.FullPath(), err) } if err != nil { @@ -104,7 +107,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo } if hasVideoJS { - switch file.Ext() { + switch *file.Ext() { case "ogg", "ogv", "mp4", "m4v", "webm", "webmv": p.VideoJSSync(resp.ImageId) } @@ -118,7 +121,7 @@ func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *piwigo for chunk := range chunks { var err error data := &url.Values{ - "original_sum": []string{file.MD5()}, + "original_sum": []string{*file.MD5()}, "position": []string{fmt.Sprint(chunk.Position)}, "type": []string{"file"}, "data": []string{chunk.Buffer.String()}, @@ -128,7 +131,7 @@ func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *piwigo if err == nil { break } - stat.Error(fmt.Sprintf("UploadChunk %d", i), file.FullPath(), err) + stat.Error(fmt.Sprintf("UploadChunk %d", i), *file.FullPath(), err) } stat.Commit(chunk.Size) if err != nil { @@ -193,7 +196,7 @@ func (p *Piwigo) ScanTree( Name: dir.Name(), CategoryId: parentCategoryId, } - if !filter.Has(file.Ext()) { + if !filter.Has(*file.Ext()) { continue } stat.Add() diff --git a/internal/piwigo/piwigotools/file_to_upload.go b/internal/piwigo/piwigotools/file_to_upload.go index d1fffe7..c01cd5e 100644 --- a/internal/piwigo/piwigotools/file_to_upload.go +++ b/internal/piwigo/piwigotools/file_to_upload.go @@ -15,9 +15,10 @@ import ( ) type FileInfo struct { - md5 string - size int64 - ext string + fullpath *string + md5 *string + size *int64 + ext *string createdAt *TimeResult } @@ -26,73 +27,119 @@ type FileToUpload struct { Name string CategoryId int - info *FileInfo + info FileInfo } -func (f *FileToUpload) FullPath() string { - return filepath.Join(f.Dir, f.Name) +func (f *FileToUpload) FullPath() *string { + if f.info.fullpath != nil { + return f.info.fullpath + } + fp := filepath.Join(f.Dir, f.Name) + f.info.fullpath = &fp + + return f.info.fullpath } -func (f *FileToUpload) Info() *FileInfo { - if f.info != nil { - return f.info +func (f *FileToUpload) Size() *int64 { + if f.info.size != nil { + return f.info.size } - file, err := os.Open(f.FullPath()) + st, err := os.Stat(*f.FullPath()) + if err != nil { + return nil + } + size := st.Size() + + f.info.size = &size + + return f.info.size +} + +func (f *FileToUpload) Ext() *string { + if f.info.ext != nil { + return f.info.ext + } + + ext := strings.ToLower(filepath.Ext(f.Name)[1:]) + f.info.ext = &ext + + return f.info.ext +} + +func (f *FileToUpload) MD5() *string { + if f.info.md5 != nil { + return f.info.md5 + } + file, err := os.Open(*f.FullPath()) if err != nil { return nil } defer file.Close() - - st, err := file.Stat() - if err != nil { - return nil - } - hash := md5.New() if _, err = io.Copy(hash, file); err != nil { return nil } - checksum := fmt.Sprintf("%x", hash.Sum(nil)) + f.info.md5 = &checksum - info := FileInfo{ - size: st.Size(), - md5: checksum, - createdAt: f.exifCreatedAt(), - } - - f.info = &info - return f.info -} - -func (f *FileToUpload) Checked() bool { - return f.info != nil -} - -func (f *FileToUpload) MD5() string { - if info := f.Info(); info != nil { - return info.md5 - } - return "" -} - -func (f *FileToUpload) Size() int64 { - if info := f.Info(); info != nil { - return info.size - } - return -1 -} - -func (f *FileToUpload) Ext() string { - return strings.ToLower(filepath.Ext(f.Name)[1:]) + return f.info.md5 } func (f *FileToUpload) CreatedAt() *TimeResult { - if info := f.Info(); info != nil { - return info.createdAt + if f.info.createdAt != nil { + return f.info.createdAt } - return nil + + et, err := exiftool.NewExiftool() + if err != nil { + return nil + } + defer et.Close() + + var createdAt *time.Time + var CreateDateFormat = "2006:01:02 15:04:05-07:00" + + fileInfos := et.ExtractMetadata(*f.FullPath()) + for _, fileInfo := range fileInfos { + if fileInfo.Err != nil { + continue + } + + var t time.Time + for k, v := range fileInfo.Fields { + switch k { + case "CreateDate": + offset, ok := fileInfo.Fields["OffsetTime"] + if !ok { + offset = "+00:00" + } + v := fmt.Sprintf("%s%s", v, offset) + t, err = time.Parse(CreateDateFormat, v) + case "CreationDate": + t, err = time.Parse(CreateDateFormat, fmt.Sprint(v)) + default: + continue + } + if err != nil { + continue + } + if createdAt == nil || createdAt.After(t) { + createdAt = &t + } + } + } + + if createdAt != nil { + result := TimeResult(*createdAt) + f.info.createdAt = &result + } + + return f.info.createdAt +} + +func (f *FileToUpload) Checked() bool { + return f.info.md5 != nil } var ( @@ -108,7 +155,7 @@ type FileToUploadChunk struct { } func (f *FileToUpload) Base64Chunker() (chan *FileToUploadChunk, error) { - fh, err := os.Open(f.FullPath()) + fh, err := os.Open(*f.FullPath()) if err != nil { return nil, err } @@ -144,49 +191,3 @@ func (f *FileToUpload) Base64Chunker() (chan *FileToUploadChunk, error) { return out, nil } - -func (f *FileToUpload) exifCreatedAt() *TimeResult { - et, err := exiftool.NewExiftool() - if err != nil { - return nil - } - defer et.Close() - - var createdAt *time.Time - var CreateDateFormat = "2006:01:02 15:04:05-07:00" - - fileInfos := et.ExtractMetadata(f.FullPath()) - for _, fileInfo := range fileInfos { - if fileInfo.Err != nil { - continue - } - - var t time.Time - for k, v := range fileInfo.Fields { - switch k { - case "CreateDate": - offset, ok := fileInfo.Fields["OffsetTime"] - if !ok { - offset = "+00:00" - } - v := fmt.Sprintf("%s%s", v, offset) - t, err = time.Parse(CreateDateFormat, v) - case "CreationDate": - t, err = time.Parse(CreateDateFormat, fmt.Sprint(v)) - default: - continue - } - if err != nil { - continue - } - if createdAt == nil || createdAt.After(t) { - createdAt = &t - } - } - } - if createdAt != nil { - result := TimeResult(*createdAt) - return &result - } - return nil -}