Index: Improve handling of archived photos, skip when possible #2257

internal/api/index.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/gin-gonic/gin"
+
 	"github.com/photoprism/photoprism/internal/acl"
 	"github.com/photoprism/photoprism/internal/entity"
 	"github.com/photoprism/photoprism/internal/event"
@@ -30,8 +31,9 @@ func StartIndexing(router *gin.RouterGroup) {
 		}
 
 		conf := service.Config()
+		settings := conf.Settings()
 
-		if !conf.Settings().Features.Library {
+		if !settings.Features.Library {
 			AbortFeatureDisabled(c)
 			return
 		}
@@ -45,12 +47,12 @@ func StartIndexing(router *gin.RouterGroup) {
 			return
 		}
 
+		// Configure index options.
 		path := conf.OriginalsPath()
+		convert := settings.Index.Convert && conf.SidecarWritable()
+		skipArchived := settings.Index.SkipArchived
 
-		ind := service.Index()
-
-		convert := conf.Settings().Index.Convert && conf.SidecarWritable()
-		indOpt := photoprism.NewIndexOptions(filepath.Clean(f.Path), f.Rescan, convert, true, false)
+		indOpt := photoprism.NewIndexOptions(filepath.Clean(f.Path), f.Rescan, convert, true, false, skipArchived)
 
 		if len(indOpt.Path) > 1 {
 			event.InfoMsg(i18n.MsgIndexingFiles, clean.Log(indOpt.Path))
@@ -58,17 +60,21 @@ func StartIndexing(router *gin.RouterGroup) {
 			event.InfoMsg(i18n.MsgIndexingOriginals)
 		}
 
+		// Start indexing.
+		ind := service.Index()
 		indexed := ind.Start(indOpt)
 
 		RemoveFromFolderCache(entity.RootOriginals)
 
-		prg := service.Purge()
-
+		// Configure purge options.
 		prgOpt := photoprism.PurgeOptions{
 			Path:   filepath.Clean(f.Path),
 			Ignore: indexed,
 		}
 
+		// Start purging.
+		prg := service.Purge()
+
 		if files, photos, err := prg.Start(prgOpt); err != nil {
 			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": txt.UpperFirst(err.Error())})
 			return

internal/auto/index.go

@@ -52,6 +52,7 @@ func Index() error {
 	}
 
 	conf := service.Config()
+	settings := conf.Settings()
 
 	start := time.Now()
 
@@ -59,8 +60,8 @@ func Index() error {
 
 	ind := service.Index()
 
-	convert := conf.Settings().Index.Convert && conf.SidecarWritable()
-	indOpt := photoprism.NewIndexOptions(entity.RootPath, false, convert, true, false)
+	convert := settings.Index.Convert && conf.SidecarWritable()
+	indOpt := photoprism.NewIndexOptions(entity.RootPath, false, convert, true, false, true)
 
 	indexed := ind.Start(indOpt)
 

internal/commands/faces.go

@@ -250,9 +250,11 @@ func facesIndexAction(ctx *cli.Context) error {
 
 	var indexed fs.Done
 
+	settings := conf.Settings()
+
 	if w := service.Index(); w != nil {
-		convert := conf.Settings().Index.Convert && conf.SidecarWritable()
-		opt := photoprism.NewIndexOptions(subPath, true, convert, true, true)
+		convert := settings.Index.Convert && conf.SidecarWritable()
+		opt := photoprism.NewIndexOptions(subPath, true, convert, true, true, true)
 
 		indexed = w.Start(opt)
 	}

internal/commands/index.go

@@ -29,7 +29,11 @@ var IndexCommand = cli.Command{
 var indexFlags = []cli.Flag{
 	cli.BoolFlag{
 		Name:  "force, f",
-		Usage: "re-index all originals, including unchanged files",
+		Usage: "rescan all originals, including unchanged files",
+	},
+	cli.BoolFlag{
+		Name:  "archived, a",
+		Usage: "update archived photos, do not skip them",
 	},
 	cli.BoolFlag{
 		Name:  "cleanup, c",
@@ -70,7 +74,7 @@ func indexAction(ctx *cli.Context) error {
 
 	if w := service.Index(); w != nil {
 		convert := conf.Settings().Index.Convert && conf.SidecarWritable()
-		opt := photoprism.NewIndexOptions(subPath, ctx.Bool("force"), convert, true, false)
+		opt := photoprism.NewIndexOptions(subPath, ctx.Bool("force"), convert, true, false, !ctx.Bool("archived"))
 
 		indexed = w.Start(opt)
 	}

internal/commands/show_config.go

@@ -17,7 +17,7 @@ var ShowConfigCommand = cli.Command{
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "md, m",
-			Usage: "renders valid Markdown",
+			Usage: "render valid Markdown",
 		},
 	},
 	Action: showConfigAction,

internal/commands/show_filters.go

@@ -17,7 +17,7 @@ var ShowFiltersCommand = cli.Command{
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "md, m",
-			Usage: "renders valid Markdown",
+			Usage: "render valid Markdown",
 		},
 	},
 	Action: showFiltersAction,

internal/commands/show_formats.go

@@ -17,11 +17,11 @@ var ShowFormatsCommand = cli.Command{
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "short, s",
-			Usage: "hides format descriptions",
+			Usage: "hide format descriptions",
 		},
 		cli.BoolFlag{
 			Name:  "md, m",
-			Usage: "renders valid Markdown",
+			Usage: "render valid Markdown",
 		},
 	},
 	Action: showFormatsAction,

internal/commands/show_tags.go

@@ -18,7 +18,7 @@ var ShowTagsCommand = cli.Command{
 	Flags: []cli.Flag{
 		cli.BoolFlag{
 			Name:  "md, m",
-			Usage: "renders valid Markdown",
+			Usage: "render valid Markdown",
 		},
 	},
 	Action: showTagsAction,

internal/config/settings_index.go

@@ -2,7 +2,8 @@ package config
 
 // IndexSettings represents indexing settings.
 type IndexSettings struct {
-	Path    string `json:"path" yaml:"Path"`
-	Convert bool   `json:"convert" yaml:"Convert"`
-	Rescan  bool   `json:"rescan" yaml:"Rescan"`
+	Path         string `json:"path" yaml:"Path"`
+	Convert      bool   `json:"convert" yaml:"Convert"`
+	Rescan       bool   `json:"rescan" yaml:"Rescan"`
+	SkipArchived bool   `json:"skipArchived" yaml:"SkipArchived"`
 }

internal/config/testdata/settings.yml

@@ -36,6 +36,7 @@ Index:
   Path: /
   Convert: true
   Rescan: false
+  SkipArchived: false
 Stack:
   UUID: true
   Meta: true

internal/entity/entity_count.go

@@ -11,14 +11,16 @@ func Count(m interface{}, keys []string, values []interface{}) int {
 		return -1
 	}
 
-	var count int
+	db, count := UnscopedDb(), 0
 
-	stmt := Db().Model(m)
+	stmt := db.Model(m)
 
+	// Compose where condition.
 	for k := range keys {
 		stmt.Where("? = ?", gorm.Expr(keys[k]), values[k])
 	}
 
+	// Fetch count from database.
 	if err := stmt.Count(&count).Error; err != nil {
 		log.Debugf("entity: %s (count records)", err)
 		return -1

internal/entity/entity_update.go

@@ -6,11 +6,15 @@ import (
 
 // Update updates an existing record in the database.
 func Update(m interface{}, keyNames ...string) (err error) {
+	// Unscoped so soft-deleted records can still be updated.
+	db := UnscopedDb()
+
 	// New entity?
-	if Db().NewRecord(m) {
+	if db.NewRecord(m) {
 		return fmt.Errorf("new record")
 	}
 
+	// Extract interface slice with all values including zero.
 	values, keys, err := ModelValues(m, keyNames...)
 
 	// Has keys and values?
@@ -20,16 +24,16 @@ func Update(m interface{}, keyNames ...string) (err error) {
 		return fmt.Errorf("record keys missing")
 	}
 
-	// Perform update.
-	res := Db().Model(m).Updates(values)
+	// Update values.
+	result := db.Model(m).Updates(values)
 
 	// Successful?
-	if res.Error != nil {
+	if result.Error != nil {
 		return err
-	} else if res.RowsAffected > 1 {
+	} else if result.RowsAffected > 1 {
 		log.Debugf("entity: updated statement affected more than one record - possible bug")
 		return nil
-	} else if res.RowsAffected == 1 {
+	} else if result.RowsAffected == 1 {
 		return nil
 	} else if Count(m, keyNames, keys) != 1 {
 		return fmt.Errorf("record not found")

internal/entity/entity_values.go

@@ -47,22 +47,31 @@ func ModelValues(m interface{}, keyNames ...string) (result Values, keys []inter
 			continue
 		}
 
-		name := field.Name
+		fieldName := field.Name
 
 		// Skip timestamps.
-		if name == "" || name == "UpdatedAt" || name == "CreatedAt" {
+		if fieldName == "" || fieldName == "UpdatedAt" || fieldName == "CreatedAt" {
 			continue
 		}
 
 		v := values.Field(i)
 
+		switch v.Kind() {
+		case reflect.Slice, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer:
+			continue
+		case reflect.Struct:
+			if v.IsZero() {
+				continue
+			}
+		}
+
 		// Skip read-only fields.
 		if !v.CanSet() {
 			continue
 		}
 
 		// Skip keys.
-		if isKey(name) {
+		if isKey(fieldName) {
 			if !v.IsZero() {
 				keys = append(keys, v.Interface())
 			}
@@ -70,7 +79,7 @@ func ModelValues(m interface{}, keyNames ...string) (result Values, keys []inter
 		}
 
 		// Add value to result.
-		result[name] = v.Interface()
+		result[fieldName] = v.Interface()
 	}
 
 	if len(result) == 0 {

internal/photoprism/import.go

@@ -102,8 +102,9 @@ func (imp *Import) Start(opt ImportOptions) fs.Done {
 
 	filesImported := 0
 
-	convert := imp.conf.Settings().Index.Convert && imp.conf.SidecarWritable()
-	indexOpt := NewIndexOptions("/", true, convert, true, false)
+	settings := imp.conf.Settings()
+	convert := settings.Index.Convert && imp.conf.SidecarWritable()
+	indexOpt := NewIndexOptions("/", true, convert, true, false, false)
 	skipRaw := imp.conf.DisableRaw()
 	ignore := fs.NewIgnoreList(fs.IgnoreFile, true, false)
 

internal/photoprism/index_mediafile.go

@@ -307,8 +307,12 @@ func (ind *Index) MediaFile(m *MediaFile, o IndexOptions, originalName, photoUID
 	}
 
 	if photo.PhotoQuality == -1 && (file.FilePrimary || fileChanged) {
-		// Restore photos that have been purged automatically.
+		// Restore pictures that have been purged automatically.
 		photo.DeletedAt = nil
+	} else if o.SkipArchived && photo.DeletedAt != nil {
+		// Skip archived pictures for faster indexing.
+		result.Status = IndexArchived
+		return result
 	}
 
 	// Extra labels to ba added when new files have a photo id.

internal/photoprism/index_options.go

@@ -7,18 +7,20 @@ type IndexOptions struct {
 	Convert         bool
 	Stack           bool
 	FacesOnly       bool
+	SkipArchived    bool
 	OriginalsLimit  int
 	ResolutionLimit int
 }
 
 // NewIndexOptions returns new index options instance.
-func NewIndexOptions(path string, rescan, convert, stack, facesOnly bool) IndexOptions {
+func NewIndexOptions(path string, rescan, convert, stack, facesOnly, skipArchived bool) IndexOptions {
 	result := IndexOptions{
 		Path:            path,
 		Rescan:          rescan,
 		Convert:         convert,
 		Stack:           stack,
 		FacesOnly:       facesOnly,
+		SkipArchived:    skipArchived,
 		OriginalsLimit:  Config().OriginalsLimit(),
 		ResolutionLimit: Config().ResolutionLimit(),
 	}
@@ -33,20 +35,20 @@ func (o *IndexOptions) SkipUnchanged() bool {
 
 // IndexOptionsAll returns new index options with all options set to true.
 func IndexOptionsAll() IndexOptions {
-	return NewIndexOptions("/", true, true, true, false)
+	return NewIndexOptions("/", true, true, true, false, true)
 }
 
 // IndexOptionsFacesOnly returns new index options for updating faces only.
 func IndexOptionsFacesOnly() IndexOptions {
-	return NewIndexOptions("/", true, true, true, true)
+	return NewIndexOptions("/", true, true, true, true, true)
 }
 
 // IndexOptionsSingle returns new index options for unstacked, single files.
 func IndexOptionsSingle() IndexOptions {
-	return NewIndexOptions("/", true, true, false, false)
+	return NewIndexOptions("/", true, true, false, false, false)
 }
 
 // IndexOptionsNone returns new index options with all options set to false.
 func IndexOptionsNone() IndexOptions {
-	return NewIndexOptions("", false, false, false, false)
+	return NewIndexOptions("", false, false, false, false, false)
 }