Indexer: Improve stacking and indexing of moved files

frontend/src/locales/de.po

@@ -1676,7 +1676,7 @@ msgstr "Ähnlich"
 
 #: src/dialog/photo/info.vue:93
 msgid "Single"
-msgstr "Einzelbild"
+msgstr "Einzelaufnahme"
 
 #: src/dialog/account/edit.vue:183 src/dialog/photo/files.vue:84
 #: src/dialog/photo/files.vue:31

internal/commands/optimize.go

@@ -40,7 +40,7 @@ func optimizeAction(ctx *cli.Context) error {
 
 	worker := workers.NewMeta(conf)
 
-	if err := worker.Start(); err != nil {
+	if err := worker.Start(time.Second * 15); err != nil {
 		return err
 	} else {
 		elapsed := time.Since(start)

internal/entity/photo_optimize.go

@@ -2,6 +2,7 @@ package entity
 
 import (
 	"errors"
+	"fmt"
 	"reflect"
 	"strings"
 	"time"
@@ -105,7 +106,7 @@ func (m *Photo) Optimize(stackMeta, stackUuid bool) (updated bool, merged Photos
 		m.UpdateLocation()
 	}
 
-	if merged, err = m.Stack(stackMeta, stackUuid); err != nil {
+	if merged, err = m.Stack(stackMeta, stackUuid, true); err != nil {
 		log.Errorf("photo: %s (stack)", err)
 	}
 
@@ -145,51 +146,66 @@ func (m *Photo) Optimize(stackMeta, stackUuid bool) (updated bool, merged Photos
 func (m *Photo) ResolvePrimary() error {
 	var file File
 
-	if err := Db().First(&file, "file_primary = 1 AND photo_id = ?", m.ID).Error; err == nil {
+	if err := Db().Where("file_primary = 1 AND photo_id = ?", m.ID).First(&file).Error; err == nil && file.ID > 0 {
 		return file.ResolvePrimary()
 	}
 
-	if err := Db().First(&file, "file_type = 'jpg' AND photo_id = ?", m.ID).Error; err == nil {
+	return nil
-		file.FilePrimary = true
-		return file.ResolvePrimary()
-	}
-
-	return m.Update("PhotoQuality", -1)
 }
 
-// Stack merges a photo with identical ones.
+// Identical returns identical photos that can be merged.
-func (m *Photo) Stack(stackMeta, stackUuid bool) (identical Photos, err error) {
+func (m *Photo) Identical(findMeta, findUuid, findOlder bool) (identical Photos, err error) {
-	if !stackMeta && !stackUuid || m.PhotoSingle || m.DeletedAt != nil {
+	if !findMeta && !findUuid || m.PhotoSingle || m.DeletedAt != nil {
 		return identical, nil
 	}
 
+	op := "<>"
+
+	if findOlder {
+		op = "<"
+	}
+
 	switch {
-	case stackMeta && stackUuid && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta && rnd.IsUUID(m.UUID):
+	case findMeta && findUuid && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta && rnd.IsUUID(m.UUID):
-		if err := Db().Where("id > ? AND photo_single = 0", m.ID).
+		if err := Db().
 			Where("(taken_at = ? AND taken_src = 'meta' AND cell_id = ? AND camera_serial = ? AND camera_id = ?) OR (uuid <> '' AND uuid = ?)",
-				m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.UUID).Find(&identical).Error; err != nil {
+				m.TakenAt, m.CellID, m.CameraSerial, m.CameraID, m.UUID).
+			Where(fmt.Sprintf("id %s ? AND photo_single = 0 AND deleted_at IS NULL", op), m.ID).
+			Order("id ASC").Find(&identical).Error; err != nil {
 			return identical, err
 		}
-	case stackMeta && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta:
+	case findMeta && m.HasLocation() && m.HasLatLng() && m.TakenSrc == SrcMeta:
-		if err := Db().Where("id > ? AND photo_single = 0", m.ID).
+		if err := Db().
 			Where("taken_at = ? AND taken_src = 'meta' AND cell_id = ? AND camera_serial = ? AND camera_id = ?",
-				m.TakenAt, m.CellID, m.CameraSerial, m.CameraID).Error; err != nil {
+				m.TakenAt, m.CellID, m.CameraSerial, m.CameraID).
+			Where(fmt.Sprintf("id %s ? AND photo_single = 0 AND deleted_at IS NULL", op), m.ID).
+			Order("id ASC").Find(&identical).Error; err != nil {
 			return identical, err
 		}
-	case stackUuid && rnd.IsUUID(m.UUID):
+	case findUuid && rnd.IsUUID(m.UUID):
-		if err := Db().Where("id > ? AND photo_single = 0", m.ID).
+		if err := Db().
-			Where("uuid <> '' AND uuid = ?", m.UUID).Error; err != nil {
+			Where(fmt.Sprintf("uuid = ? AND id %s ? AND photo_single = 0 AND deleted_at IS NULL", op), m.UUID, m.ID).
+			Order("id ASC").Find(&identical).Error; err != nil {
 			return identical, err
 		}
-	default:
-		return identical, nil
 	}
 
-	if len(identical) == 0 {
+	return identical, nil
-		return identical, nil
+}
+
+// Stack merges a photo with identical ones.
+func (m *Photo) Stack(stackMeta, stackUuid, stackOlder bool) (identical Photos, err error) {
+	identical, err = m.Identical(stackMeta, stackUuid, stackOlder)
+
+	if len(identical) == 0 || err != nil {
+		return identical, err
 	}
 
 	for _, photo := range identical {
+		if photo.DeletedAt != nil || photo.ID == m.ID {
+			continue
+		}
+
 		if err := UnscopedDb().Exec("UPDATE `files` SET photo_id = ?, photo_uid = ?, file_primary = 0 WHERE photo_id = ?", m.ID, m.PhotoUID, photo.ID).Error; err != nil {
 			return identical, err
 		}
@@ -207,9 +223,14 @@ func (m *Photo) Stack(stackMeta, stackUuid bool) (identical Photos, err error) {
 			log.Warnf("photo: unknown SQL dialect (stack)")
 		}
 
-		if err := photo.Updates(map[string]interface{}{"DeletedAt": Timestamp(), "PhotoQuality": -1}); err != nil {
+		deleted := Timestamp()
+
+		if err := UnscopedDb().Exec("UPDATE `photos` SET photo_quality = -1, deleted_at = ? WHERE id = ?", Timestamp(), photo.ID).Error; err != nil {
 			return identical, err
 		}
+
+		photo.DeletedAt = &deleted
+		photo.PhotoQuality = -1
 	}
 
 	return identical, err

internal/entity/photo_test.go

@@ -881,7 +881,7 @@ func TestPhoto_Approve(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		assert.Equal(t, 3, photo.PhotoQuality)
+		assert.Equal(t, 4, photo.PhotoQuality)
 	})
 	t.Run("quality = 1", func(t *testing.T) {
 		photo := Photo{PhotoQuality: 1}

internal/photoprism/index_mediafile.go

@@ -175,10 +175,10 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 	// Look for existing photo if file wasn't indexed yet...
 	if !fileExists {
 		fullBase := m.BasePrefix(false)
-		photoQuery = entity.UnscopedDb().First(&photo, "photo_path = ? AND photo_name IN (?)", filePath, []string{fullBase, fileBase})
 
-		if photoQuery.Error == nil {
+		if photoQuery = entity.UnscopedDb().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fileBase); photoQuery.Error == nil || fileBase == fullBase {
-			fileBase = photo.PhotoName
+			// Skip next query.
+		} else if photoQuery = entity.UnscopedDb().First(&photo, "photo_path = ? AND photo_name = ?", filePath, fullBase); photoQuery.Error == nil {
 			fileStacked = true
 		}
 
@@ -258,7 +258,7 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 	// Flag first JPEG as primary file for this photo.
 	if !file.FilePrimary {
 		if photoExists {
-			if q := entity.Db().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
+			if q := entity.UnscopedDb().Where("file_type = 'jpg' AND file_primary = 1 AND photo_id = ?", photo.ID).First(&primaryFile); q.Error != nil {
 				file.FilePrimary = m.IsJpeg()
 			}
 		} else {
@@ -815,8 +815,13 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName string) (
 		log.Errorf("index: %s in %s (set download id)", err, logName)
 	}
 
-	// Write YAML sidecar file (optional).
+	if stacked, err := photo.Stack(Config().Settings().StackMeta(), Config().Settings().StackUUID(), true); err != nil {
-	if file.FilePrimary && Config().SidecarYaml() {
+		log.Errorf("index: %s in %s (stack)", err.Error(), logName)
+	} else if len(stacked) > 0 {
+		log.Infof("index: merged %s with existing stack", logName)
+		result.Status = IndexStacked
+	} else if file.FilePrimary && Config().SidecarYaml() {
+		// Write YAML sidecar file (optional).
 		yamlFile := photo.YamlFileName(Config().OriginalsPath(), Config().SidecarPath())
 
 		if err := photo.SaveAsYaml(yamlFile); err != nil {

internal/photoprism/mediafile.go

@@ -1006,10 +1006,12 @@ func (m *MediaFile) ResampleDefault(thumbPath string, force bool) (err error) {
 func (m *MediaFile) RenameSidecars(oldFileName string) (renamed map[string]string, err error) {
 	renamed = make(map[string]string)
 
-	newName := m.RelPrefix(Config().OriginalsPath(), false)
+	sidecarPath := Config().SidecarPath()
+	originalsPath := Config().OriginalsPath()
 
-	oldPrefix := fs.RelPrefix(oldFileName, Config().OriginalsPath(), false)
+	newName := m.RelPrefix(originalsPath, false)
-	globPrefix := filepath.Join(Config().SidecarPath(), oldPrefix) + "."
+	oldPrefix := fs.RelPrefix(oldFileName, originalsPath, false)
+	globPrefix := filepath.Join(sidecarPath, oldPrefix) + "."
 
 	matches, err := filepath.Glob(regexp.QuoteMeta(globPrefix) + "*")
 
@@ -1018,12 +1020,25 @@ func (m *MediaFile) RenameSidecars(oldFileName string) (renamed map[string]strin
 	}
 
 	for _, srcName := range matches {
-		destName := filepath.Join(Config().SidecarPath(), newName+filepath.Ext(srcName))
+		destName := filepath.Join(sidecarPath, newName+filepath.Ext(srcName))
+
+		if fs.FileExists(destName) {
+			renamed[fs.RelName(srcName, sidecarPath)] = fs.RelName(destName, sidecarPath)
+
+			if err := os.Remove(srcName); err != nil {
+				log.Errorf("media: failed removing sidecar %s", txt.Quote(fs.RelName(srcName, sidecarPath)))
+			} else {
+				log.Infof("media: removed sidecar %s", txt.Quote(fs.RelName(srcName, sidecarPath)))
+			}
+
+			continue
+		}
 
 		if err := fs.Move(srcName, destName); err != nil {
 			return renamed, err
 		} else {
-			renamed[fs.RelName(srcName, Config().SidecarPath())] = fs.RelName(destName, Config().SidecarPath())
+			log.Infof("media: moved existing sidecar to %s", txt.Quote(newName+filepath.Ext(srcName)))
+			renamed[fs.RelName(srcName, sidecarPath)] = fs.RelName(destName, sidecarPath)
 		}
 	}
 

internal/query/files.go

@@ -98,7 +98,7 @@ func RenameFile(srcRoot, srcName, destRoot, destName string) error {
 		return fmt.Errorf("can't rename %s/%s to %s/%s", srcRoot, srcName, destRoot, destName)
 	}
 
-	return Db().Exec("UPDATE files SET file_root = ?, file_name = ?, file_missing = ?, deleted_at = NULL WHERE file_root = ? AND file_name = ?", destRoot, destName, srcRoot, srcName).Error
+	return Db().Exec("UPDATE files SET file_root = ?, file_name = ?, file_missing = 0, deleted_at = NULL WHERE file_root = ? AND file_name = ?", destRoot, destName, srcRoot, srcName).Error
 }
 
 // SetPhotoPrimary sets a new primary image file for a photo.

internal/query/photo.go

@@ -99,7 +99,7 @@ func ResetPhotoQuality() error {
 }
 
 // PhotosCheck returns photos selected for maintenance.
-func PhotosCheck(limit int, offset int) (entities entity.Photos, err error) {
+func PhotosCheck(limit, offset int, delay time.Duration) (entities entity.Photos, err error) {
 	err = Db().
 		Preload("Labels", func(db *gorm.DB) *gorm.DB {
 			return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
@@ -112,7 +112,7 @@ func PhotosCheck(limit int, offset int) (entities entity.Photos, err error) {
 		Preload("Cell").
 		Preload("Cell.Place").
 		Where("checked_at IS NULL OR checked_at < ?", time.Now().Add(-1*time.Hour*24*3)).
-		Where("updated_at < ? OR (cell_id = 'zz' AND photo_lat <> 0)", time.Now().Add(-1*time.Minute*10)).
+		Where("updated_at < ? OR (cell_id = 'zz' AND photo_lat <> 0)", time.Now().Add(-1*delay)).
 		Order("photos.ID ASC").Limit(limit).Offset(offset).Find(&entities).Error
 
 	return entities, err

internal/query/photo_test.go

@@ -2,6 +2,7 @@ package query
 
 import (
 	"testing"
+	"time"
 
 	"github.com/photoprism/photoprism/internal/entity"
 
@@ -72,7 +73,7 @@ func TestResetPhotosQuality(t *testing.T) {
 }
 
 func TestPhotosCheck(t *testing.T) {
-	result, err := PhotosCheck(10, 0)
+	result, err := PhotosCheck(10, 0, time.Second)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/workers/meta.go

@@ -30,7 +30,7 @@ func (worker *Meta) originalsPath() string {
 }
 
 // Start starts the metadata worker.
-func (worker *Meta) Start() (err error) {
+func (worker *Meta) Start(delay time.Duration) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("metadata: %s (panic)\nstack: %s", r, debug.Stack())
@@ -53,7 +53,7 @@ func (worker *Meta) Start() (err error) {
 	optimized := 0
 
 	for {
-		photos, err := query.PhotosCheck(limit, offset)
+		photos, err := query.PhotosCheck(limit, offset, delay)
 
 		if err != nil {
 			return err

internal/workers/meta_test.go

@@ -3,6 +3,7 @@ package workers
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/mutex"
@@ -22,13 +23,13 @@ func TestPrism_Start(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := worker.Start(); err == nil {
+	if err := worker.Start(time.Second); err == nil {
 		t.Fatal("error expected")
 	}
 
 	mutex.MetaWorker.Stop()
 
-	if err := worker.Start(); err != nil {
+	if err := worker.Start(time.Second); err != nil {
 		t.Fatal(err)
 	}
 }

internal/workers/workers.go

@@ -44,7 +44,7 @@ func StartMeta(conf *config.Config) {
 	if !mutex.WorkersBusy() {
 		go func() {
 			worker := NewMeta(conf)
-			if err := worker.Start(); err != nil {
+			if err := worker.Start(time.Minute); err != nil {
 				log.Warnf("metadata: %s", err)
 			}
 		}()