johanels/photoprism
1
0
Fork 0
mirror of https://github.com/photoprism/photoprism.git synced 2025-12-12 00:34:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

Indexer: Improve stacking and indexing of moved files

Browse Source
This commit is contained in:
Michael Mayer
2020-12-11 22:09:11 +01:00
parent 8a86fbc60f
commit 9f4066edb6
14 changed files with 92 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
frontend/src/locales/de.mo
View File

Binary file not shown.

2
frontend/src/locales/de.po
View File

@@ -1676,7 +1676,7 @@ msgstr "Ähnlich"
#: src/dialog/photo/info.vue:93
msgid "Single"
msgstr "Einzelbild"
msgstr "Einzelaufnahme"
#: src/dialog/account/edit.vue:183 src/dialog/photo/files.vue:84
#: src/dialog/photo/files.vue:31

2
frontend/src/locales/translations.json
View File

File diff suppressed because one or more lines are too long

2
internal/commands/optimize.go
View File

@@ -40,7 +40,7 @@ func optimizeAction(ctx *cli.Context) error {
worker := workers.NewMeta(conf)
if err := worker.Start(); err != nil {
if err := worker.Start(time.Second * 15); err != nil {
return err
} else {
elapsed := time.Since(start)

71
internal/entity/photo_optimize.go
View File

@@ -2,6 +2,7 @@ package entity
import (
"errors"
"fmt"
"reflect"
"strings"
"time"
@@ -105,7 +106,7 @@ func (m *Photo) Optimize(stackMeta, stackUuid bool) (updated bool, merged Photos
m.UpdateLocation()
}
if merged, err = m.Stack(stackMeta, stackUuid); err != nil {
if merged, err = m.Stack(stackMeta, stackUuid, true); err != nil {
log.Errorf("photo: %s (stack)", err)
}
@@ -145,51 +146,66 @@ func (m *Photo) Optimize(stackMeta, stackUuid bool) (updated bool, merged Photos
func (m *Photo) ResolvePrimary() error {
var file File
if err := Db().First(&file, "file_primary = 1 AND photo_id = ?", m.ID).Error; err == nil {
if err := Db().Where("file_primary = 1 AND photo_id = ?", m.ID).First(&file).Error; err == nil && file.ID > 0 {
return file.ResolvePrimary()
}
if err := Db().First(&file, "file_type = 'jpg' AND photo_id = ?", m.ID).Error; err == nil {
file.FilePrimary = true
return file.ResolvePrimary()
}
return m.Update("PhotoQuality", -1)
return nil
}
// Stack merges a photo with identical ones.
func (m *Photo) Stack(stackMeta, stackUuid bool) (identical Photos, err error) {
if !stackMeta && !stackUuid || m.PhotoSingle || m.DeletedAt != nil {
// Identical returns identical photos that can be merged.
func (m *Photo) Identical(findMeta, findUuid, findOlder bool) (identical Photos, err error) {
if !findMeta && !findUuid || m.PhotoSingle || m.DeletedAt != nil {
return identical, nil
}
op := "<>"
if findOlder {
op = "<"
}
switch {
case stackMeta && stackUuid && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta && rnd.IsUUID(m.UUID):
if err := Db().Where("id > ? AND photo_single = 0", m.ID).
case findMeta && findUuid && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta && rnd.IsUUID(m.UUID):
if err := Db().
Where("(taken_at = ? AND taken_src = 'meta' AND cell_id = ? AND camera_serial = ? AND camera_id = ?) OR (uuid <> '' AND uuid = ?)",
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.UUID).Find(&identical).Error; err != nil {
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.UUID).
Where(fmt.Sprintf("id %s ? AND photo_single = 0 AND deleted_at IS NULL", op), m.ID).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
case stackMeta && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta:
if err := Db().Where("id > ? AND photo_single = 0", m.ID).
case findMeta && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta:
if err := Db().
Where("taken_at = ? AND taken_src = 'meta' AND cell_id = ? AND camera_serial = ? AND camera_id = ?",
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID).Error; err != nil {
m.TakenAt, m.CellID, m.CameraSerial, m.CameraID).
Where(fmt.Sprintf("id %s ? AND photo_single = 0 AND deleted_at IS NULL", op), m.ID).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
case stackUuid && rnd.IsUUID(m.UUID):
if err := Db().Where("id > ? AND photo_single = 0", m.ID).
Where("uuid <> '' AND uuid = ?", m.UUID).Error; err != nil {
case findUuid && rnd.IsUUID(m.UUID):
if err := Db().
Where(fmt.Sprintf("uuid = ? AND id %s ? AND photo_single = 0 AND deleted_at IS NULL", op), m.UUID, m.ID).
Order("id ASC").Find(&identical).Error; err != nil {
return identical, err
}
default:
return identical, nil
}
if len(identical) == 0 {
return identical, nil
return identical, nil
}
// Stack merges a photo with identical ones.
func (m *Photo) Stack(stackMeta, stackUuid, stackOlder bool) (identical Photos, err error) {
identical, err = m.Identical(stackMeta, stackUuid, stackOlder)
if len(identical) == 0 || err != nil {
return identical, err
}
for _, photo := range identical {
if photo.DeletedAt != nil || photo.ID == m.ID {
continue
}
if err := UnscopedDb().Exec("UPDATE `files` SET photo_id = ?, photo_uid = ?, file_primary = 0 WHERE photo_id = ?", m.ID, m.PhotoUID, photo.ID).Error; err != nil {
return identical, err
}
@@ -207,9 +223,14 @@ func (m *Photo) Stack(stackMeta, stackUuid bool) (identical Photos, err error) {
log.Warnf("photo: unknown SQL dialect (stack)")
}
if err := photo.Updates(map[string]interface{}{"DeletedAt": Timestamp(), "PhotoQuality": -1}); err != nil {
deleted := Timestamp()
if err := UnscopedDb().Exec("UPDATE `photos` SET photo_quality = -1, deleted_at = ? WHERE id = ?", Timestamp(), photo.ID).Error; err != nil {
return identical, err
}
photo.DeletedAt = &deleted
photo.PhotoQuality = -1
}
return identical, err

2
internal/entity/photo_test.go
View File

@@ -881,7 +881,7 @@ func TestPhoto_Approve(t *testing.T) {
t.Fatal(err)
}
assert.Equal(t, 3, photo.PhotoQuality)
assert.Equal(t, 4, photo.PhotoQuality)
})
t.Run("quality = 1", func(t *testing.T) {
photo := Photo{PhotoQuality: 1}

17
internal/photoprism/index_mediafile.go
View File

@@ -175,10 +175,10 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
// Look for existing photo if file wasn't indexed yet...
if !fileExists {
fullBase := m.BasePrefix(false)
photoQuery = entity.UnscopedDb().First(&photo, "photo_path = ? AND photo_name IN (?)", filePath, []string{fullBase, fileBase})
if photoQuery.Error == nil {
fileBase = photo.PhotoName
if photoQuery = entity.UnscopedDb().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase); photoQuery.Error == nil || fileBase == fullBase {
// Skip next query.
} else if photoQuery = entity.UnscopedDb().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fullBase); photoQuery.Error == nil {
fileStacked = true
}
@@ -258,7 +258,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
// Flag first JPEG as primary file for this photo.
if !file.FilePrimary {
if photoExists {
if q := entity.Db().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
if q := entity.UnscopedDb().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
file.FilePrimary = m.IsJpeg()
}
} else {
@@ -815,8 +815,13 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
log.Errorf("index: %s in %s (set download id)", err, logName)
}
// Write YAML sidecar file (optional).
if file.FilePrimary && Config().SidecarYaml() {
if stacked, err := photo.Stack(Config().Settings().StackMeta(), Config().Settings().StackUUID(), true); err != nil {
log.Errorf("index: %s in %s (stack)", err.Error(), logName)
} else if len(stacked) > 0 {
log.Infof("index: merged %s with existing stack", logName)
result.Status = IndexStacked
} else if file.FilePrimary && Config().SidecarYaml() {
// Write YAML sidecar file (optional).
yamlFile := photo.YamlFileName(Config().OriginalsPath(), Config().SidecarPath())
if err := photo.SaveAsYaml(yamlFile); err != nil {

25
internal/photoprism/mediafile.go
View File

@@ -1006,10 +1006,12 @@ func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) {
func (m *MediaFile) RenameSidecars(oldFileName string) (renamed map[string]string, err error) {
renamed = make(map[string]string)
newName := m.RelPrefix(Config().OriginalsPath(), false)
sidecarPath := Config().SidecarPath()
originalsPath := Config().OriginalsPath()
oldPrefix := fs.RelPrefix(oldFileName, Config().OriginalsPath(), false)
globPrefix := filepath.Join(Config().SidecarPath(), oldPrefix) + "."
newName := m.RelPrefix(originalsPath, false)
oldPrefix := fs.RelPrefix(oldFileName, originalsPath, false)
globPrefix := filepath.Join(sidecarPath, oldPrefix) + "."
matches, err := filepath.Glob(regexp.QuoteMeta(globPrefix) + "*")
@@ -1018,12 +1020,25 @@ func (m *MediaFile) RenameSidecars(oldFileName string) (renamed map[string]strin
}
for _, srcName := range matches {
destName := filepath.Join(Config().SidecarPath(), newName+filepath.Ext(srcName))
destName := filepath.Join(sidecarPath, newName+filepath.Ext(srcName))
if fs.FileExists(destName) {
renamed[fs.RelName(srcName, sidecarPath)] = fs.RelName(destName, sidecarPath)
if err := os.Remove(srcName); err != nil {
log.Errorf("media: failed removing sidecar %s", txt.Quote(fs.RelName(srcName, sidecarPath)))
} else {
log.Infof("media: removed sidecar %s", txt.Quote(fs.RelName(srcName, sidecarPath)))
}
continue
}
if err := fs.Move(srcName, destName); err != nil {
return renamed, err
} else {
renamed[fs.RelName(srcName, Config().SidecarPath())] = fs.RelName(destName, Config().SidecarPath())
log.Infof("media: moved existing sidecar to %s", txt.Quote(newName+filepath.Ext(srcName)))
renamed[fs.RelName(srcName, sidecarPath)] = fs.RelName(destName, sidecarPath)
}
}

2
internal/query/files.go
View File

@@ -98,7 +98,7 @@ func RenameFile(srcRoot, srcName, destRoot, destName string) error {
return fmt.Errorf("can't rename %s/%s to %s/%s", srcRoot, srcName, destRoot, destName)
}
return Db().Exec("UPDATE files SET file_root = ?, file_name = ?, file_missing = ?, deleted_at = NULL WHERE file_root = ? AND file_name = ?", destRoot, destName, srcRoot, srcName).Error
return Db().Exec("UPDATE files SET file_root = ?, file_name = ?, file_missing = 0, deleted_at = NULL WHERE file_root = ? AND file_name = ?", destRoot, destName, srcRoot, srcName).Error
}
// SetPhotoPrimary sets a new primary image file for a photo.

4
internal/query/photo.go
View File

@@ -99,7 +99,7 @@ func ResetPhotoQuality() error {
}
// PhotosCheck returns photos selected for maintenance.
func PhotosCheck(limit int, offset int) (entities entity.Photos, err error) {
func PhotosCheck(limit, offset int, delay time.Duration) (entities entity.Photos, err error) {
err = Db().
Preload("Labels", func(db *gorm.DB) *gorm.DB {
return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
@@ -112,7 +112,7 @@ func PhotosCheck(limit int, offset int) (entities entity.Photos, err error) {
Preload("Cell").
Preload("Cell.Place").
Where("checked_at IS NULL OR checked_at < ?", time.Now().Add(-1*time.Hour*24*3)).
Where("updated_at < ? OR (cell_id = 'zz' AND photo_lat <> 0)", time.Now().Add(-1*time.Minute*10)).
Where("updated_at < ? OR (cell_id = 'zz' AND photo_lat <> 0)", time.Now().Add(-1*delay)).
Order("photos.ID ASC").Limit(limit).Offset(offset).Find(&entities).Error
return entities, err

3
internal/query/photo_test.go
View File

@@ -2,6 +2,7 @@ package query
import (
"testing"
"time"
"github.com/photoprism/photoprism/internal/entity"
@@ -72,7 +73,7 @@ func TestResetPhotosQuality(t *testing.T) {
}
func TestPhotosCheck(t *testing.T) {
result, err := PhotosCheck(10, 0)
result, err := PhotosCheck(10, 0, time.Second)
if err != nil {
t.Fatal(err)
}

4
internal/workers/meta.go
View File

@@ -30,7 +30,7 @@ func (worker *Meta) originalsPath() string {
}
// Start starts the metadata worker.
func (worker *Meta) Start() (err error) {
func (worker *Meta) Start(delay time.Duration) (err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("metadata: %s (panic)\nstack: %s", r, debug.Stack())
@@ -53,7 +53,7 @@ func (worker *Meta) Start() (err error) {
optimized := 0
for {
photos, err := query.PhotosCheck(limit, offset)
photos, err := query.PhotosCheck(limit, offset, delay)
if err != nil {
return err

5
internal/workers/meta_test.go
View File

@@ -3,6 +3,7 @@ package workers
import (
"strings"
"testing"
"time"
"github.com/photoprism/photoprism/internal/config"
"github.com/photoprism/photoprism/internal/mutex"
@@ -22,13 +23,13 @@ func TestPrism_Start(t *testing.T) {
t.Fatal(err)
}
if err := worker.Start(); err == nil {
if err := worker.Start(time.Second); err == nil {
t.Fatal("error expected")
}
mutex.MetaWorker.Stop()
if err := worker.Start(); err != nil {
if err := worker.Start(time.Second); err != nil {
t.Fatal(err)
}
}

2
internal/workers/workers.go
View File

@@ -44,7 +44,7 @@ func StartMeta(conf *config.Config) {
if !mutex.WorkersBusy() {
go func() {
worker := NewMeta(conf)
if err := worker.Start(); err != nil {
if err := worker.Start(time.Minute); err != nil {
log.Warnf("metadata: %s", err)
}
}()