celogeek/piwigo-cli
1
0
Fork 0
mirror of https://github.com/celogeek/piwigo-cli.git synced 2025-05-25 10:12:37 +02:00
Code Issues Projects Releases Wiki Activity

merge file info

Browse Source
This commit is contained in:
Celogeek 2021-12-31 16:26:31 +01:00
parent 28e99a59d2
commit cabd7084fc
Signed by: celogeek
GPG Key ID: E6B7BDCFC446233A
5 changed files with 167 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
internal/base64/chunker.go
View File

@ -1,55 +0,0 @@
package base64
import (
"bytes"
b64 "encoding/base64"
"os"
)
var CHUNK_SIZE int64 = 1 * 1024 * 1024
var CHUNK_BUFF_SIZE int64 = 32 * 1024
var CHUNK_BUFF_COUNT = CHUNK_SIZE / CHUNK_BUFF_SIZE
type Chunk struct {
Position int64
Size int64
Buffer bytes.Buffer
}
func Chunker(filename string) (chan *Chunk, error) {
f, err := os.Open(filename)
if err != nil {
return nil, err
}
out := make(chan *Chunk, 8)
chunker := func() {
b := make([]byte, CHUNK_BUFF_SIZE)
defer f.Close()
defer close(out)
ok := false
for position := int64(0); !ok; position += 1 {
bf := &Chunk{
Position: position,
}
b64 := b64.NewEncoder(b64.StdEncoding, &bf.Buffer)
for i := int64(0); i < CHUNK_BUFF_COUNT; i++ {
n, _ := f.Read(b)
if n == 0 {
ok = true
break
}
bf.Size += int64(n)
b64.Write(b[:n])
}
b64.Close()
if bf.Size > 0 {
out <- bf
}
}
}
go chunker()
return out, nil
}

56
internal/exif/extract.go
View File

@ -1,56 +0,0 @@
package exif
import (
"fmt"
"time"
"github.com/barasher/go-exiftool"
)
type Info struct {
CreatedAt *time.Time
}
var (
CreateDateFormat = "2006:01:02 15:04:05-07:00"
)
func Extract(filename string) (*Info, error) {
et, err := exiftool.NewExiftool()
if err != nil {
return nil, err
}
defer et.Close()
var resp *Info = &Info{}
fileInfos := et.ExtractMetadata(filename)
for _, fileInfo := range fileInfos {
if fileInfo.Err != nil {
continue
}
var t time.Time
for k, v := range fileInfo.Fields {
switch k {
case "CreateDate":
offset, ok := fileInfo.Fields["OffsetTime"]
if !ok {
offset = "+00:00"
}
v := fmt.Sprintf("%s%s", v, offset)
t, err = time.Parse(CreateDateFormat, v)
case "CreationDate":
t, err = time.Parse(CreateDateFormat, fmt.Sprint(v))
default:
continue
}
if err != nil {
continue
}
if resp.CreatedAt == nil || resp.CreatedAt.After(t) {
resp.CreatedAt = &t
}
}
}
return resp, nil
}

22
internal/md5/file.go
View File

@ -1,22 +0,0 @@
package md5
import (
"crypto/md5"
"fmt"
"io"
"os"
)
func File(filename string) (string, error) {
file, err := os.Open(filename)
if err != nil {
return "", err
}
defer file.Close()
hash := md5.New()
if _, err = io.Copy(hash, file); err != nil {
return "", err
}
return fmt.Sprintf("%x", hash.Sum(nil)), nil
}

13
internal/piwigo/files.go
View File

@ -8,8 +8,6 @@ import (
"strings"
"sync"
"github.com/celogeek/piwigo-cli/internal/base64"
"github.com/celogeek/piwigo-cli/internal/exif"
"github.com/celogeek/piwigo-cli/internal/piwigo/piwigotools"
"golang.org/x/text/unicode/norm"
)
@ -59,7 +57,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo
return
}
wg := &sync.WaitGroup{}
chunks, err := base64.Chunker(file.FullPath())
chunks, err := file.Base64Chunker()
if err != nil {
stat.Error("Base64Chunker", file.FullPath(), err)
return
@ -82,10 +80,9 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo
data.Set("original_sum", file.MD5())
data.Set("original_filename", file.Name)
data.Set("check_uniqueness", "true")
info, _ := exif.Extract(file.FullPath())
if info != nil && info.CreatedAt != nil {
data.Set("date_creation", piwigotools.TimeResult(*info.CreatedAt).String())
if file.CreatedAt() != nil {
fmt.Println(file.CreatedAt())
data.Set("date_creation", file.CreatedAt().String())
}
if file.CategoryId > 0 {
data.Set("categories", fmt.Sprint(file.CategoryId))
@ -117,7 +114,7 @@ func (p *Piwigo) Upload(file *piwigotools.FileToUpload, stat *piwigotools.FileTo
stat.Done()
}
func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *base64.Chunk, wg *sync.WaitGroup, stat *piwigotools.FileToUploadStat, ok *bool) {
func (p *Piwigo) UploadChunk(file *piwigotools.FileToUpload, chunks chan *piwigotools.FileToUploadChunk, wg *sync.WaitGroup, stat *piwigotools.FileToUploadStat, ok *bool) {
defer wg.Done()
for chunk := range chunks {
var err error

186
internal/piwigo/piwigotools/file_to_upload.go
View File

@ -1,58 +1,196 @@
package piwigotools
import (
"bytes"
"crypto/md5"
"encoding/base64"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/celogeek/piwigo-cli/internal/md5"
"github.com/barasher/go-exiftool"
)
type FileInfo struct {
md5 string
size int64
ext string
createdAt *TimeResult
}
type FileToUpload struct {
Dir string
Name string
CategoryId int
md5 *string
size *int64
ext *string
info *FileInfo
}
func (f *FileToUpload) FullPath() string {
return filepath.Join(f.Dir, f.Name)
}
func (f *FileToUpload) Info() *FileInfo {
if f.info != nil {
return f.info
}
file, err := os.Open(f.FullPath())
if err != nil {
return nil
}
defer file.Close()
st, err := file.Stat()
if err != nil {
return nil
}
hash := md5.New()
if _, err = io.Copy(hash, file); err != nil {
return nil
}
checksum := fmt.Sprintf("%x", hash.Sum(nil))
info := FileInfo{
size: st.Size(),
ext: strings.ToLower(filepath.Ext(f.Name)[1:]),
md5: checksum,
createdAt: f.exifCreatedAt(),
}
f.info = &info
return f.info
}
func (f *FileToUpload) Checked() bool {
return f.md5 != nil
return f.info != nil
}
func (f *FileToUpload) MD5() string {
if f.md5 == nil {
md5, err := md5.File(f.FullPath())
if err != nil {
return ""
}
f.md5 = &md5
if info := f.Info(); info != nil {
return info.md5
}
return *f.md5
return ""
}
func (f *FileToUpload) Size() int64 {
if f.size == nil {
st, err := os.Stat(f.FullPath())
if err != nil {
return -1
}
size := st.Size()
f.size = &size
if info := f.Info(); info != nil {
return info.size
}
return *f.size
return -1
}
func (f *FileToUpload) Ext() string {
if f.ext == nil {
ext := strings.ToLower(filepath.Ext(f.Name)[1:])
f.ext = &ext
if info := f.Info(); info != nil {
return info.ext
}
return *f.ext
return ""
}
func (f *FileToUpload) CreatedAt() *TimeResult {
if info := f.Info(); info != nil {
return info.createdAt
}
return nil
}
var (
CHUNK_SIZE int64 = 1 * 1024 * 1024
CHUNK_BUFF_SIZE int64 = 32 * 1024
CHUNK_BUFF_COUNT = CHUNK_SIZE / CHUNK_BUFF_SIZE
)
type FileToUploadChunk struct {
Position int64
Size int64
Buffer bytes.Buffer
}
func (f *FileToUpload) Base64Chunker() (chan *FileToUploadChunk, error) {
fh, err := os.Open(f.FullPath())
if err != nil {
return nil, err
}
out := make(chan *FileToUploadChunk, 8)
chunker := func() {
b := make([]byte, CHUNK_BUFF_SIZE)
defer fh.Close()
defer close(out)
ok := false
for position := int64(0); !ok; position += 1 {
bf := &FileToUploadChunk{
Position: position,
}
b64 := base64.NewEncoder(base64.StdEncoding, &bf.Buffer)
for i := int64(0); i < CHUNK_BUFF_COUNT; i++ {
n, _ := fh.Read(b)
if n == 0 {
ok = true
break
}
bf.Size += int64(n)
b64.Write(b[:n])
}
b64.Close()
if bf.Size > 0 {
out <- bf
}
}
}
go chunker()
return out, nil
}
func (f *FileToUpload) exifCreatedAt() *TimeResult {
et, err := exiftool.NewExiftool()
if err != nil {
return nil
}
defer et.Close()
var createdAt *time.Time
var CreateDateFormat = "2006:01:02 15:04:05-07:00"
fileInfos := et.ExtractMetadata(f.FullPath())
for _, fileInfo := range fileInfos {
if fileInfo.Err != nil {
continue
}
var t time.Time
for k, v := range fileInfo.Fields {
switch k {
case "CreateDate":
offset, ok := fileInfo.Fields["OffsetTime"]
if !ok {
offset = "+00:00"
}
v := fmt.Sprintf("%s%s", v, offset)
t, err = time.Parse(CreateDateFormat, v)
case "CreationDate":
t, err = time.Parse(CreateDateFormat, fmt.Sprint(v))
default:
continue
}
if err != nil {
continue
}
if createdAt == nil || createdAt.After(t) {
createdAt = &t
}
}
}
if createdAt != nil {
result := TimeResult(*createdAt)
return &result
}
return nil
}