split tools

This commit is contained in:
Celogeek 2021-12-31 12:48:43 +01:00
parent 653bf3ffdd
commit 6a3bac66d9
Signed by: celogeek
GPG Key ID: E6B7BDCFC446233A
4 changed files with 77 additions and 62 deletions

View File

@ -1,8 +1,8 @@
package piwigotools package base64
import ( import (
"bytes" "bytes"
"encoding/base64" b64 "encoding/base64"
"os" "os"
) )
@ -10,29 +10,29 @@ var CHUNK_SIZE int64 = 1 * 1024 * 1024
var CHUNK_BUFF_SIZE int64 = 32 * 1024 var CHUNK_BUFF_SIZE int64 = 32 * 1024
var CHUNK_BUFF_COUNT = CHUNK_SIZE / CHUNK_BUFF_SIZE var CHUNK_BUFF_COUNT = CHUNK_SIZE / CHUNK_BUFF_SIZE
type Base64Chunk struct { type Chunk struct {
Position int64 Position int64
Size int64 Size int64
Buffer bytes.Buffer Buffer bytes.Buffer
} }
func Base64Chunker(filename string) (out chan *Base64Chunk, err error) { func Chunker(filename string) (chan *Chunk, error) {
f, err := os.Open(filename) f, err := os.Open(filename)
if err != nil { if err != nil {
return return nil, err
} }
out = make(chan *Base64Chunk, 8) out := make(chan *Chunk, 8)
go func() { chunker := func() {
b := make([]byte, CHUNK_BUFF_SIZE) b := make([]byte, CHUNK_BUFF_SIZE)
defer f.Close() defer f.Close()
defer close(out) defer close(out)
ok := false ok := false
for position := int64(0); !ok; position += 1 { for position := int64(0); !ok; position += 1 {
bf := &Base64Chunk{ bf := &Chunk{
Position: position, Position: position,
} }
b64 := base64.NewEncoder(base64.StdEncoding, &bf.Buffer) b64 := b64.NewEncoder(b64.StdEncoding, &bf.Buffer)
for i := int64(0); i < CHUNK_BUFF_COUNT; i++ { for i := int64(0); i < CHUNK_BUFF_COUNT; i++ {
n, _ := f.Read(b) n, _ := f.Read(b)
if n == 0 { if n == 0 {
@ -47,7 +47,9 @@ func Base64Chunker(filename string) (out chan *Base64Chunk, err error) {
out <- bf out <- bf
} }
} }
}() }
return go chunker()
return out, nil
} }

56
internal/exif/extract.go Normal file
View File

@ -0,0 +1,56 @@
package exif
import (
"fmt"
"time"
"github.com/barasher/go-exiftool"
)
type Info struct {
CreatedAt *time.Time
}
var (
CreateDateFormat = "2006:01:02 15:04:05-07:00"
)
func Extract(filename string) (*Info, error) {
et, err := exiftool.NewExiftool()
if err != nil {
return nil, err
}
defer et.Close()
var resp *Info = &Info{}
fileInfos := et.ExtractMetadata(filename)
for _, fileInfo := range fileInfos {
if fileInfo.Err != nil {
continue
}
var t time.Time
for k, v := range fileInfo.Fields {
switch k {
case "CreateDate":
offset, ok := fileInfo.Fields["OffsetTime"]
if !ok {
offset = "+00:00"
}
v := fmt.Sprintf("%s%s", v, offset)
t, err = time.Parse(CreateDateFormat, v)
case "CreationDate":
t, err = time.Parse(CreateDateFormat, fmt.Sprint(v))
default:
continue
}
if err != nil {
continue
}
if resp.CreatedAt == nil || resp.CreatedAt.After(t) {
resp.CreatedAt = &t
}
}
}
return resp, nil
}

View File

@ -8,6 +8,8 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/celogeek/piwigo-cli/internal/base64"
"github.com/celogeek/piwigo-cli/internal/exif"
"github.com/celogeek/piwigo-cli/internal/piwigo/piwigotools" "github.com/celogeek/piwigo-cli/internal/piwigo/piwigotools"
"golang.org/x/text/unicode/norm" "golang.org/x/text/unicode/norm"
) )
@ -57,7 +59,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo
return return
} }
wg := &sync.WaitGroup{} wg := &sync.WaitGroup{}
chunks, err := piwigotools.Base64Chunker(file.FullPath()) chunks, err := base64.Chunker(file.FullPath())
if err != nil { if err != nil {
stat.Error("Base64Chunker", file.FullPath(), err) stat.Error("Base64Chunker", file.FullPath(), err)
return return
@ -75,14 +77,15 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo
// lock this process for committing the file // lock this process for committing the file
exif, _ := piwigotools.Exif(file.FullPath())
var resp *FileUploadResult var resp *FileUploadResult
data := &url.Values{} data := &url.Values{}
data.Set("original_sum", file.MD5()) data.Set("original_sum", file.MD5())
data.Set("original_filename", file.Name) data.Set("original_filename", file.Name)
data.Set("check_uniqueness", "true") data.Set("check_uniqueness", "true")
if exif != nil && exif.CreatedAt != nil {
data.Set("date_creation", exif.CreatedAt.String()) info, _ := exif.Extract(file.FullPath())
if info != nil && info.CreatedAt != nil {
data.Set("date_creation", piwigotools.TimeResult(*info.CreatedAt).String())
} }
if file.CategoryId > 0 { if file.CategoryId > 0 {
data.Set("categories", fmt.Sprint(file.CategoryId)) data.Set("categories", fmt.Sprint(file.CategoryId))
@ -114,7 +117,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo
stat.Done() stat.Done()
} }
func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *piwigotools.Base64Chunk, wg *sync.WaitGroup, stat *piwigotools.FileToUploadStat, ok *bool) { func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *base64.Chunk, wg *sync.WaitGroup, stat *piwigotools.FileToUploadStat, ok *bool) {
defer wg.Done() defer wg.Done()
for chunk := range chunks { for chunk := range chunks {
var err error var err error

View File

@ -1,46 +0,0 @@
package piwigotools
import (
"fmt"
"time"
"github.com/barasher/go-exiftool"
)
type ExifResult struct {
CreatedAt *TimeResult
}
func Exif(filename string) (*ExifResult, error) {
et, err := exiftool.NewExiftool()
if err != nil {
return nil, err
}
defer et.Close()
resp := &ExifResult{}
fileInfos := et.ExtractMetadata(filename)
for _, fileInfo := range fileInfos {
if fileInfo.Err != nil {
fmt.Printf("Error concerning %v: %v\n", fileInfo.File, fileInfo.Err)
continue
}
for k, v := range fileInfo.Fields {
switch k {
case "CreateDate", "CreationDate":
switch v := v.(type) {
case string:
t, err := time.Parse("2006:01:02 15:04:05-07:00", v)
if err == nil {
if resp.CreatedAt == nil || time.Time(*resp.CreatedAt).After(t) {
r := TimeResult(t)
resp.CreatedAt = &r
}
}
}
}
}
}
return resp, nil
}