Places: Remove unique label index and purge unused location infos #1664

internal/commands/optimize.go

@@ -7,6 +7,7 @@ import (
 	"github.com/urfave/cli"
 
 	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/entity"
 	"github.com/photoprism/photoprism/internal/service"
 	"github.com/photoprism/photoprism/internal/workers"
 )
@@ -47,7 +48,15 @@ func optimizeAction(ctx *cli.Context) error {
 	force := ctx.Bool("force")
 	worker := workers.NewMeta(conf)
 
-	if err := worker.Start(time.Second*15, force); err != nil {
+	delay := 15 * time.Second
+	interval := entity.MetadataUpdateInterval
+
+	if force {
+		delay = 0
+		interval = 0
+	}
+
+	if err := worker.Start(delay, interval, force); err != nil {
 		return err
 	} else {
 		elapsed := time.Since(start)

internal/entity/db.go

@@ -22,45 +22,6 @@ type DbProvider interface {
 	Db() *gorm.DB
 }
 
-// RecreateTable drops and recreates the database table for a clean start.
-func RecreateTable(models ...interface{}) (err error) {
-	n := len(models)
-
-	// Return if no models were provided.
-	if n < 1 {
-		return nil
-	}
-
-	// Drop existing tables.
-	if err = Db().DropTable(models...).Error; err != nil {
-		return fmt.Errorf("%s (drop table)", err)
-	}
-
-	time.Sleep(100 * time.Millisecond)
-
-	done := 0
-
-	// Create dropped tables.
-	for i := 0; i < 60; i++ {
-		for m := range models {
-			if err = Db().CreateTable(models[m]).Error; err != nil {
-				log.Debugf("entity: %s (create table)", err)
-			} else {
-				done++
-			}
-
-			if done >= n {
-				return err
-			}
-		}
-
-		// Wait 3 seconds before trying again...
-		time.Sleep(3 * time.Second)
-	}
-
-	return err
-}
-
 // IsDialect returns true if the given sql dialect is used.
 func IsDialect(name string) bool {
 	return name == Db().Dialect().GetName()

internal/entity/db_test.go

@@ -1,7 +1,6 @@
 package entity
 
 import (
-	"testing"
 	"time"
 )
 
@@ -18,15 +17,3 @@ type TestEntity struct {
 func (TestEntity) TableName() string {
 	return "test_ignore"
 }
-
-func TestRecreateTable(t *testing.T) {
-	t.Run("TestEntity", func(t *testing.T) {
-		if err := Db().CreateTable(TestEntity{}).Error; err != nil {
-			t.Fatal(err)
-		}
-
-		if err := RecreateTable(TestEntity{}); err != nil {
-			t.Fatal(err)
-		}
-	})
-}

internal/entity/entity.go

@@ -146,6 +146,13 @@ func CreateDefaultFixtures() {
 	CreateUnknownLens()
 }
 
+// MigrateIndexes runs additional table index migration queries.
+func MigrateIndexes() {
+	if err := Db().Exec("DROP INDEX IF EXISTS idx_places_place_label ON places").Error; err != nil {
+		log.Errorf("%s: %s (drop index)", DbDialect(), err)
+	}
+}
+
 // MigrateDb creates database tables and inserts default fixtures as needed.
 func MigrateDb(dropDeprecated bool) {
 	if dropDeprecated {
@@ -155,6 +162,8 @@ func MigrateDb(dropDeprecated bool) {
 	Entities.Migrate()
 	Entities.WaitForMigration()
 
+	MigrateIndexes()
+
 	CreateDefaultFixtures()
 }
 

internal/hub/config.go

@@ -130,9 +130,11 @@ func (c *Config) Refresh() (err error) {
 	}
 
 	c.Sanitize()
+
 	client := &http.Client{Timeout: 60 * time.Second}
 	url := ServiceURL
 	method := http.MethodPost
+
 	var req *http.Request
 
 	if c.Key != "" {

internal/hub/feedback.go

@@ -53,6 +53,7 @@ func (c *Config) SendFeedback(f form.Feedback) (err error) {
 	client := &http.Client{Timeout: 60 * time.Second}
 	url := fmt.Sprintf(FeedbackURL, c.Key)
 	method := http.MethodPost
+
 	var req *http.Request
 
 	log.Debugf("sending feedback to %s", ApiHost())

internal/photoprism/places.go

@@ -59,11 +59,6 @@ func (w *Places) Start() (updated []string, err error) {
 		return []string{}, nil
 	}
 
-	// Drop and recreate places database table.
-	if err = entity.RecreateTable(entity.Place{}); err != nil {
-		return []string{}, fmt.Errorf("index: %s", err)
-	}
-
 	// List of updated cells.
 	updated = make([]string, 0, len(cells))
 
@@ -97,6 +92,11 @@ func (w *Places) Start() (updated []string, err error) {
 		time.Sleep(33 * time.Millisecond)
 	}
 
+	// Remove unused entries from the places table.
+	if err := query.PurgePlaces(); err != nil {
+		log.Errorf("index: %s (purge places)", err)
+	}
+
 	// Update location-related photo metadata in the index.
 	if _, err := w.UpdatePhotos(); err != nil {
 		log.Errorf("index: %s (update photos)", err)

internal/photoprism/places_test.go

@@ -21,10 +21,7 @@ func TestPlaces(t *testing.T) {
 		}
 
 		t.Logf("updated: %#v", updated)
-	})
 
-	t.Run("UpdatePhotos", func(t *testing.T) {
-		w := NewPlaces(config.TestConfig())
 		affected, err := w.UpdatePhotos()
 
 		if err != nil {

internal/photoprism/purge.go

@@ -270,6 +270,11 @@ func (w *Purge) Start(opt PurgeOptions) (purgedFiles map[string]bool, purgedPhot
 		log.Errorf("index: %s (update album entries)", err)
 	}
 
+	// Remove unused entries from the places table.
+	if err := query.PurgePlaces(); err != nil {
+		log.Errorf("index: %s (purge places)", err)
+	}
+
 	// Update precalculated photo and file counts.
 	if err := entity.UpdateCounts(); err != nil {
 		log.Warnf("index: %s (update counts)", err)

internal/query/photo.go

@@ -85,7 +85,7 @@ func PhotosMissing(limit int, offset int) (entities entity.Photos, err error) {
 }
 
 // PhotosMetadataUpdate returns photos selected for metadata maintenance.
-func PhotosMetadataUpdate(limit, offset int, delay time.Duration) (entities entity.Photos, err error) {
+func PhotosMetadataUpdate(limit, offset int, delay, interval time.Duration) (entities entity.Photos, err error) {
 	err = Db().
 		Preload("Labels", func(db *gorm.DB) *gorm.DB {
 			return db.Order("photos_labels.uncertainty ASC, photos_labels.label_id DESC")
@@ -97,7 +97,7 @@ func PhotosMetadataUpdate(limit, offset int, delay time.Duration) (entities enti
 		Preload("Place").
 		Preload("Cell").
 		Preload("Cell.Place").
-		Where("checked_at IS NULL OR checked_at < ?", time.Now().Add(-1*entity.MetadataUpdateInterval)).
+		Where("checked_at IS NULL OR checked_at < ?", time.Now().Add(-1*interval)).
 		Where("updated_at < ? OR (cell_id = 'zz' AND photo_lat <> 0)", time.Now().Add(-1*delay)).
 		Order("photos.ID ASC").Limit(limit).Offset(offset).Find(&entities).Error
 

internal/query/photo_test.go

@@ -68,7 +68,8 @@ func TestMissingPhotos(t *testing.T) {
 }
 
 func TestPhotosMetadataUpdate(t *testing.T) {
-	result, err := PhotosMetadataUpdate(10, 0, time.Second)
+	interval := entity.MetadataUpdateInterval
+	result, err := PhotosMetadataUpdate(10, 0, time.Second, interval)
 
 	if err != nil {
 		t.Fatal(err)

internal/query/places.go

@@ -27,3 +27,11 @@ func CellIDs() (result Cells, err error) {
 
 	return result, err
 }
+
+// PurgePlaces removes unused entries from the places table.
+func PurgePlaces() error {
+	query := "DELETE FROM places WHERE id NOT IN (SELECT DISTINCT place_id FROM cells)" +
+		" AND id NOT IN (SELECT DISTINCT place_id FROM photos)"
+
+	return Db().Exec(query).Error
+}

internal/query/places_test.go

@@ -15,3 +15,10 @@ func TestCellIDs(t *testing.T) {
 		t.Logf("cell count: %v", len(result))
 	})
 }
+func TestPurgePlaces(t *testing.T) {
+	t.Run("success", func(t *testing.T) {
+		if err := PurgePlaces(); err != nil {
+			t.Fatal(err)
+		}
+	})
+}

internal/workers/meta.go

@@ -30,7 +30,7 @@ func (m *Meta) originalsPath() string {
 }
 
 // Start metadata optimization routine.
-func (m *Meta) Start(delay time.Duration, force bool) (err error) {
+func (m *Meta) Start(delay, interval time.Duration, force bool) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("metadata: %s (panic)\nstack: %s", r, debug.Stack())
@@ -64,7 +64,7 @@ func (m *Meta) Start(delay time.Duration, force bool) (err error) {
 
 	// Run index optimization.
 	for {
-		photos, err := query.PhotosMetadataUpdate(limit, offset, delay)
+		photos, err := query.PhotosMetadataUpdate(limit, offset, delay, interval)
 
 		if err != nil {
 			return err

internal/workers/meta_test.go

@@ -5,9 +5,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/mutex"
-	"github.com/stretchr/testify/assert"
 )
 
 func TestPrism_Start(t *testing.T) {
@@ -23,13 +24,16 @@ func TestPrism_Start(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := worker.Start(time.Second, true); err == nil {
+	delay := time.Second
+	interval := time.Second
+
+	if err := worker.Start(delay, interval, true); err == nil {
 		t.Fatal("error expected")
 	}
 
 	mutex.MetaWorker.Stop()
 
-	if err := worker.Start(time.Second, true); err != nil {
+	if err := worker.Start(delay, interval, true); err != nil {
 		t.Fatal(err)
 	}
 }

internal/workers/workers.go

@@ -35,6 +35,7 @@ import (
 	"time"
 
 	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/entity"
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/mutex"
 )
@@ -83,7 +84,11 @@ func StartMeta(conf *config.Config) {
 	if !mutex.WorkersBusy() {
 		go func() {
 			worker := NewMeta(conf)
-			if err := worker.Start(time.Minute, false); err != nil {
+
+			delay := time.Minute
+			interval := entity.MetadataUpdateInterval
+
+			if err := worker.Start(delay, interval, false); err != nil {
 				log.Warnf("metadata: %s", err)
 			}
 		}()