WebDAV: Changes trigger auto indexing / importing #281

The safety delay may be configured individually using
PHOTOPRISM_AUTO_INDEX and PHOTOPRISM_AUTO_IMPORT. A negative value
disables the feature.

internal/auto/auto.go

@@ -0,0 +1,83 @@
+/*
+
+Package workers contains auto indexing & importing workers.
+
+Copyright (c) 2018 - 2021 Michael Mayer <hello@photoprism.org>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+    PhotoPrism® is a registered trademark of Michael Mayer.  You may use it as required
+    to describe our software, run your own server, for educational purposes, but not for
+    offering commercial goods, products, or services without prior written permission.
+    In other words, please ask.
+
+Feel free to send an e-mail to hello@photoprism.org if you have questions,
+want to support our work, or just want to say hello.
+
+Additional information can be found in our Developer Guide:
+https://docs.photoprism.org/developer-guide/
+
+*/
+package auto
+
+import (
+	"time"
+
+	"github.com/photoprism/photoprism/internal/config"
+
+	"github.com/photoprism/photoprism/internal/event"
+)
+
+var log = event.Log
+
+var stop = make(chan bool, 1)
+
+// Wait starts waiting for indexing & importing opportunities.
+func Start(conf *config.Config) {
+	// Don't start ticker if both are disabled.
+	if conf.AutoIndex().Seconds() <= 0 && conf.AutoImport().Seconds() <= 0 {
+		return
+	}
+
+	ticker := time.NewTicker(time.Minute)
+
+	go func() {
+		for {
+			select {
+			case <-stop:
+				ticker.Stop()
+				return
+			case <-ticker.C:
+				if mustIndex(conf.AutoIndex()) {
+					log.Debugf("auto-index: starting")
+					ResetIndex()
+					if err := Index(); err != nil {
+						log.Errorf("auto-index: %s", err)
+					}
+				} else if mustImport(conf.AutoImport()) {
+					log.Debugf("auto-import: starting")
+					ResetImport()
+					if err := Import(); err != nil {
+						log.Errorf("auto-import: %s", err)
+					}
+				}
+			}
+		}
+	}()
+}
+
+// Stop stops waiting for indexing & importing opportunities.
+func Stop() {
+	stop <- true
+}

internal/auto/auto_test.go

@@ -0,0 +1,47 @@
+package auto
+
+import (
+	"os"
+	"testing"
+
+	"github.com/photoprism/photoprism/internal/config"
+	"github.com/sirupsen/logrus"
+)
+
+func TestMain(m *testing.M) {
+	log = logrus.StandardLogger()
+	log.SetLevel(logrus.DebugLevel)
+
+	if err := os.Remove(".test.db"); err == nil {
+		log.Debugln("removed .test.db")
+	}
+
+	c := config.TestConfig()
+
+	code := m.Run()
+
+	_ = c.CloseDb()
+
+	os.Exit(code)
+}
+
+func TestStart(t *testing.T) {
+	conf := config.TestConfig()
+
+	Start(conf)
+	ShouldIndex()
+	ShouldImport()
+
+	if mustIndex(conf.AutoIndex()) {
+		t.Error("mustIndex() must return false")
+	}
+
+	if mustImport(conf.AutoImport()) {
+		t.Error("mustImport() must return false")
+	}
+
+	ResetImport()
+	ResetIndex()
+
+	Stop()
+}

internal/auto/import.go

@@ -0,0 +1,95 @@
+package auto
+
+import (
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/photoprism/photoprism/internal/api"
+	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/internal/i18n"
+	"github.com/photoprism/photoprism/internal/mutex"
+	"github.com/photoprism/photoprism/internal/photoprism"
+	"github.com/photoprism/photoprism/internal/service"
+	"github.com/photoprism/photoprism/pkg/txt"
+)
+
+var autoImport = time.Time{}
+var importMutex = sync.Mutex{}
+
+// ResetImport resets the auto import trigger time.
+func ResetImport() {
+	importMutex.Lock()
+	defer importMutex.Unlock()
+
+	autoImport = time.Time{}
+}
+
+// ShouldImport sets the auto import trigger to the current time.
+func ShouldImport() {
+	importMutex.Lock()
+	defer importMutex.Unlock()
+
+	autoImport = time.Now()
+}
+
+// mustImport tests if auto import must be started.
+func mustImport(delay time.Duration) bool {
+	if delay.Seconds() <= 0 {
+		return false
+	}
+
+	importMutex.Lock()
+	defer importMutex.Unlock()
+
+	return !autoImport.IsZero() && autoImport.Sub(time.Now()) < -1*delay && !mutex.MainWorker.Busy()
+}
+
+// Import starts importing originals e.g. after WebDAV uploads.
+func Import() error {
+	if mutex.MainWorker.Busy() {
+		return nil
+	}
+
+	conf := service.Config()
+
+	if conf.ReadOnly() || !conf.Settings().Features.Import {
+		return nil
+	}
+
+	start := time.Now()
+
+	path := filepath.Clean(conf.ImportPath())
+
+	imp := service.Import()
+
+	api.ClearFoldersCache(entity.RootImport)
+
+	event.InfoMsg(i18n.MsgCopyingFilesFrom, txt.Quote(filepath.Base(path)))
+	opt := photoprism.ImportOptionsCopy(path)
+
+	imported := imp.Start(opt)
+
+	if len(imported) == 0 {
+		return nil
+	}
+
+	moments := service.Moments()
+
+	if err := moments.Start(); err != nil {
+		log.Warnf("moments: %s", err)
+	}
+
+	elapsed := int(time.Since(start).Seconds())
+
+	msg := i18n.Msg(i18n.MsgImportCompletedIn, elapsed)
+
+	event.Success(msg)
+	event.Publish("import.completed", event.Data{"path": path, "seconds": elapsed})
+	event.Publish("index.completed", event.Data{"path": path, "seconds": elapsed})
+
+	api.UpdateClientConfig()
+
+	return nil
+}

internal/auto/index.go

@@ -0,0 +1,110 @@
+package auto
+
+import (
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/photoprism/photoprism/internal/api"
+	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/internal/i18n"
+	"github.com/photoprism/photoprism/internal/mutex"
+	"github.com/photoprism/photoprism/internal/photoprism"
+	"github.com/photoprism/photoprism/internal/service"
+)
+
+var autoIndex = time.Time{}
+var indexMutex = sync.Mutex{}
+
+// ResetIndex resets the auto index trigger time.
+func ResetIndex() {
+	indexMutex.Lock()
+	defer indexMutex.Unlock()
+
+	autoIndex = time.Time{}
+}
+
+// ShouldIndex sets the auto index trigger to the current time.
+func ShouldIndex() {
+	indexMutex.Lock()
+	defer indexMutex.Unlock()
+
+	autoIndex = time.Now()
+}
+
+// mustIndex tests if auto indexing must be started.
+func mustIndex(delay time.Duration) bool {
+	if delay.Seconds() <= 0 {
+		return false
+	}
+
+	indexMutex.Lock()
+	defer indexMutex.Unlock()
+
+	return !autoIndex.IsZero() && autoIndex.Sub(time.Now()) < -1*delay && !mutex.MainWorker.Busy()
+}
+
+// Index starts indexing originals e.g. after WebDAV uploads.
+func Index() error {
+	if mutex.MainWorker.Busy() {
+		return nil
+	}
+
+	conf := service.Config()
+
+	start := time.Now()
+
+	path := conf.OriginalsPath()
+
+	ind := service.Index()
+
+	indOpt := photoprism.IndexOptions{
+		Rescan:  false,
+		Convert: conf.Settings().Index.Convert && conf.SidecarWritable(),
+		Path:    entity.RootPath,
+		Stack:   true,
+	}
+
+	indexed := ind.Start(indOpt)
+
+	if len(indexed) == 0 {
+		return nil
+	}
+
+	api.ClearFoldersCache(entity.RootOriginals)
+
+	prg := service.Purge()
+
+	prgOpt := photoprism.PurgeOptions{
+		Path:   filepath.Clean(entity.RootPath),
+		Ignore: indexed,
+	}
+
+	if files, photos, err := prg.Start(prgOpt); err != nil {
+		return err
+	} else if len(files) > 0 || len(photos) > 0 {
+		event.InfoMsg(i18n.MsgRemovedFilesAndPhotos, len(files), len(photos))
+	}
+
+	event.Publish("index.updating", event.Data{
+		"step": "moments",
+	})
+
+	moments := service.Moments()
+
+	if err := moments.Start(); err != nil {
+		log.Warnf("moments: %s", err)
+	}
+
+	elapsed := int(time.Since(start).Seconds())
+
+	msg := i18n.Msg(i18n.MsgIndexingCompletedIn, elapsed)
+
+	event.Success(msg)
+	event.Publish("index.completed", event.Data{"path": path, "seconds": elapsed})
+
+	api.UpdateClientConfig()
+
+	return nil
+}

internal/commands/config.go

@@ -58,6 +58,8 @@ func configAction(ctx *cli.Context) error {
 	// Workers.
 	fmt.Printf("%-25s %d\n", "workers", conf.Workers())
 	fmt.Printf("%-25s %d\n", "wakeup-interval", conf.WakeupInterval()/time.Second)
+	fmt.Printf("%-25s %d\n", "auto-index", conf.AutoIndex()/time.Second)
+	fmt.Printf("%-25s %d\n", "auto-import", conf.AutoImport()/time.Second)
 
 	// Disable features.
 	fmt.Printf("%-25s %t\n", "disable-backups", conf.DisableBackups())

internal/commands/start.go

@@ -9,6 +9,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/photoprism/photoprism/internal/auto"
+
 	"github.com/photoprism/photoprism/internal/photoprism"
 
 	"github.com/photoprism/photoprism/internal/config"
@@ -117,6 +119,7 @@ func startAction(ctx *cli.Context) error {
 
 	// start share & sync workers
 	workers.Start(conf)
+	auto.Start(conf)
 
 	// set up proper shutdown of daemon and web server
 	quit := make(chan os.Signal)
@@ -126,6 +129,7 @@ func startAction(ctx *cli.Context) error {
 
 	// stop share & sync workers
 	workers.Stop()
+	auto.Stop()
 
 	log.Info("shutting down...")
 	conf.Shutdown()

internal/config/config.go

@@ -346,15 +346,37 @@ func (c *Config) Workers() int {
 	return 1
 }
 
-// WakeupInterval returns the background worker wakeup interval.
+// WakeupInterval returns the background worker wakeup interval duration.
 func (c *Config) WakeupInterval() time.Duration {
-	if c.options.WakeupInterval <= 0 {
+	if c.options.WakeupInterval <= 0 || c.options.WakeupInterval > 86400 {
 		return 15 * time.Minute
 	}
 
 	return time.Duration(c.options.WakeupInterval) * time.Second
 }
 
+// AutoIndex returns the auto indexing delay duration.
+func (c *Config) AutoIndex() time.Duration {
+	if c.options.AutoIndex < 0 {
+		return time.Duration(0)
+	} else if c.options.AutoIndex == 0 || c.options.AutoIndex > 86400 {
+		return c.WakeupInterval()
+	}
+
+	return time.Duration(c.options.AutoIndex) * time.Second
+}
+
+// AutoImport returns the auto importing delay duration.
+func (c *Config) AutoImport() time.Duration {
+	if c.options.AutoImport < 0 || c.ReadOnly() {
+		return time.Duration(0)
+	} else if c.options.AutoImport == 0 || c.options.AutoImport > 86400 {
+		return c.AutoIndex()
+	}
+
+	return time.Duration(c.options.AutoImport) * time.Second
+}
+
 // GeoApi returns the preferred geo coding api (none or places).
 func (c *Config) GeoApi() string {
 	if c.options.DisablePlaces {

internal/config/config_test.go

@@ -264,6 +264,16 @@ func TestConfig_WakeupInterval(t *testing.T) {
 	assert.Equal(t, time.Duration(900000000000), c.WakeupInterval())
 }
 
+func TestConfig_AutoIndex(t *testing.T) {
+	c := NewConfig(CliTestContext())
+	assert.Equal(t, time.Duration(0), c.AutoIndex())
+}
+
+func TestConfig_AutoImport(t *testing.T) {
+	c := NewConfig(CliTestContext())
+	assert.Equal(t, 2*time.Hour, c.AutoImport())
+}
+
 func TestConfig_GeoApi(t *testing.T) {
 	c := NewConfig(CliTestContext())
 

internal/config/flags.go

@@ -103,6 +103,16 @@ var GlobalFlags = []cli.Flag{
 		Usage:  "background worker wakeup interval in `SECONDS`",
 		EnvVar: "PHOTOPRISM_WAKEUP_INTERVAL",
 	},
+	cli.IntFlag{
+		Name:   "auto-index",
+		Usage:  "auto indexing safety delay in `SECONDS` (WebDAV)",
+		EnvVar: "PHOTOPRISM_AUTO_INDEX",
+	},
+	cli.IntFlag{
+		Name:   "auto-import",
+		Usage:  "auto importing safety delay in `SECONDS` (WebDAV)",
+		EnvVar: "PHOTOPRISM_AUTO_IMPORT",
+	},
 	cli.BoolFlag{
 		Name:   "disable-backups",
 		Usage:  "don't backup photo and album metadata to YAML files",

internal/config/options.go

@@ -52,6 +52,8 @@ type Options struct {
 	CachePath         string `yaml:"CachePath" json:"-" flag:"cache-path"`
 	Workers           int    `yaml:"Workers" json:"Workers" flag:"workers"`
 	WakeupInterval    int    `yaml:"WakeupInterval" json:"WakeupInterval" flag:"wakeup-interval"`
+	AutoIndex         int    `yaml:"AutoIndex" json:"AutoIndex" flag:"auto-index"`
+	AutoImport        int    `yaml:"AutoImport" json:"AutoImport" flag:"auto-import"`
 	DisableBackups    bool   `yaml:"DisableBackups" json:"DisableBackups" flag:"disable-backups"`
 	DisableWebDAV     bool   `yaml:"DisableWebDAV" json:"DisableWebDAV" flag:"disable-webdav"`
 	DisableSettings   bool   `yaml:"DisableSettings" json:"-" flag:"disable-settings"`

internal/config/test.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"os"
 	"path/filepath"
+	"strconv"
 	"sync"
 	"testing"
 	"time"
@@ -60,6 +61,8 @@ func NewTestOptions() *Options {
 		DetectNSFW:     true,
 		UploadNSFW:     false,
 		AssetsPath:     assetsPath,
+		AutoIndex:      -1,
+		AutoImport:     7200,
 		StoragePath:    testDataPath,
 		CachePath:      testDataPath + "/cache",
 		OriginalsPath:  testDataPath + "/originals",
@@ -167,6 +170,8 @@ func CliTestContext() *cli.Context {
 	globalSet.String("darktable-cli", config.DarktableBin, "doc")
 	globalSet.String("admin-password", config.DarktableBin, "doc")
 	globalSet.Bool("detect-nsfw", config.DetectNSFW, "doc")
+	globalSet.Int("auto-index", config.AutoIndex, "doc")
+	globalSet.Int("auto-import", config.AutoImport, "doc")
 
 	app := cli.NewApp()
 	app.Version = "0.0.0"
@@ -185,6 +190,8 @@ func CliTestContext() *cli.Context {
 	LogError(c.Set("darktable-cli", config.DarktableBin))
 	LogError(c.Set("admin-password", config.AdminPassword))
 	LogError(c.Set("detect-nsfw", "true"))
+	LogError(c.Set("auto-index", strconv.Itoa(config.AutoIndex)))
+	LogError(c.Set("auto-import", strconv.Itoa(config.AutoImport)))
 
 	return c
 }

internal/photoprism/mediafile_meta.go

@@ -94,7 +94,7 @@ func (m *MediaFile) MetaData() (result meta.Data) {
 			}
 
 			if jsonErr := m.ReadExifToolJson(); jsonErr != nil {
-				log.Debug(err)
+				log.Debug(jsonErr)
 			} else {
 				err = nil
 			}

internal/server/routes.go

@@ -10,6 +10,10 @@ import (
 )
 
 func registerRoutes(router *gin.Engine, conf *config.Config) {
+	// Enables automatic redirection if the current route can't be matched but a
+	// handler for the path with (without) the trailing slash exists.
+	router.RedirectTrailingSlash = true
+
 	// Static assets like js, css and font files.
 	router.Static("/static", conf.StaticPath())
 	router.StaticFile("/favicon.ico", filepath.Join(conf.ImgPath(), "favicon.ico"))
@@ -134,12 +138,12 @@ func registerRoutes(router *gin.Engine, conf *config.Config) {
 	if conf.DisableWebDAV() {
 		log.Info("webdav: server disabled")
 	} else {
-		WebDAV(conf.OriginalsPath(), router.Group("/originals", BasicAuth()), conf)
+		WebDAV(conf.OriginalsPath(), router.Group(WebDAVOriginals, BasicAuth()), conf)
-		log.Info("webdav: /originals/ enabled, waiting for requests")
+		log.Infof("webdav: %s/ enabled, waiting for requests", WebDAVOriginals)
 
 		if conf.ImportPath() != "" {
-			WebDAV(conf.ImportPath(), router.Group("/import", BasicAuth()), conf)
+			WebDAV(conf.ImportPath(), router.Group(WebDAVImport, BasicAuth()), conf)
-			log.Info("webdav: /import/ enabled, waiting for requests")
+			log.Infof("webdav: %s/ enabled, waiting for requests", WebDAVImport)
 		}
 	}
 

internal/server/webdav.go

@@ -4,10 +4,14 @@ import (
 	"net/http"
 
 	"github.com/gin-gonic/gin"
+	"github.com/photoprism/photoprism/internal/auto"
 	"github.com/photoprism/photoprism/internal/config"
 	"golang.org/x/net/webdav"
 )
 
+const WebDAVOriginals = "/originals"
+const WebDAVImport = "/import"
+
 // ANY /webdav/*
 func WebDAV(path string, router *gin.RouterGroup, conf *config.Config) {
 	if router == nil {
@@ -28,9 +32,28 @@ func WebDAV(path string, router *gin.RouterGroup, conf *config.Config) {
 		LockSystem: webdav.NewMemLS(),
 		Logger: func(r *http.Request, err error) {
 			if err != nil {
-				log.Printf("webdav: %s %s, ERROR: %s\n", r.Method, r.URL, err)
+				switch r.Method {
+				case "POST", "DELETE", "PUT", "MKCOL", "COPY", "MOVE", "LOCK", "UNLOCK":
+					log.Errorf("webdav-error: %s in %s %s", err, r.Method, r.URL)
+				case "PROPFIND":
+					log.Tracef("webdav-error: %s in %s %s", err, r.Method, r.URL)
+				default:
+					log.Debugf("webdav-error: %s in %s %s", err, r.Method, r.URL)
+				}
+
 			} else {
-				log.Printf("webdav: %s %s \n", r.Method, r.URL)
+				switch r.Method {
+				case "POST", "DELETE", "PUT", "COPY", "MOVE":
+					log.Infof("webdav: %s %s", r.Method, r.URL)
+
+					if router.BasePath() == WebDAVOriginals {
+						auto.ShouldIndex()
+					} else if router.BasePath() == WebDAVImport {
+						auto.ShouldImport()
+					}
+				default:
+					log.Tracef("webdav: %s %s", r.Method, r.URL)
+				}
 			}
 		},
 	}

internal/workers/workers.go

@@ -1,3 +1,34 @@
+/*
+
+Package workers contains background workers for file sync & metadata optimization.
+
+Copyright (c) 2018 - 2021 Michael Mayer <hello@photoprism.org>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+    PhotoPrism® is a registered trademark of Michael Mayer.  You may use it as required
+    to describe our software, run your own server, for educational purposes, but not for
+    offering commercial goods, products, or services without prior written permission.
+    In other words, please ask.
+
+Feel free to send an e-mail to hello@photoprism.org if you have questions,
+want to support our work, or just want to say hello.
+
+Additional information can be found in our Developer Guide:
+https://docs.photoprism.org/developer-guide/
+
+*/
 package workers
 
 import (