diff --git a/docker/photoprism-arm64/Dockerfile b/docker/photoprism-arm64/Dockerfile index 0237bfc2b..a291f782d 100644 --- a/docker/photoprism-arm64/Dockerfile +++ b/docker/photoprism-arm64/Dockerfile @@ -142,6 +142,7 @@ ENV PHOTOPRISM_ORIGINALS_PATH /photoprism/originals ENV PHOTOPRISM_IMPORT_PATH /photoprism/import ENV PHOTOPRISM_EXPORT_PATH /photoprism/export ENV PHOTOPRISM_DATABASE_PATH /photoprism/database +ENV PHOTOPRISM_TEMP_PATH /photoprism/temp ENV PHOTOPRISM_CACHE_PATH /photoprism/cache ENV PHOTOPRISM_CONFIG_PATH /photoprism/config ENV PHOTOPRISM_CONFIG_FILE /photoprism/config/photoprism.yml @@ -161,6 +162,7 @@ RUN mkdir -p \ /photoprism/import \ /photoprism/export \ /photoprism/database \ + /photoprism/temp \ /photoprism/cache RUN chmod -R 777 /photoprism diff --git a/docker/photoprism/Dockerfile b/docker/photoprism/Dockerfile index 0b8b1b9de..d098b6af9 100644 --- a/docker/photoprism/Dockerfile +++ b/docker/photoprism/Dockerfile @@ -58,6 +58,7 @@ ENV PHOTOPRISM_ORIGINALS_PATH /photoprism/originals ENV PHOTOPRISM_IMPORT_PATH /photoprism/import ENV PHOTOPRISM_EXPORT_PATH /photoprism/export ENV PHOTOPRISM_DATABASE_PATH /photoprism/database +ENV PHOTOPRISM_TEMP_PATH /photoprism/temp ENV PHOTOPRISM_CACHE_PATH /photoprism/cache ENV PHOTOPRISM_CONFIG_PATH /photoprism/config ENV PHOTOPRISM_CONFIG_FILE /photoprism/config/photoprism.yml @@ -77,6 +78,7 @@ RUN mkdir -p \ /photoprism/import \ /photoprism/export \ /photoprism/database \ + /photoprism/temp \ /photoprism/cache RUN chmod -R 777 /photoprism diff --git a/internal/commands/config.go b/internal/commands/config.go index 5be310016..bbc4dbe18 100644 --- a/internal/commands/config.go +++ b/internal/commands/config.go @@ -45,6 +45,7 @@ func configAction(ctx *cli.Context) error { fmt.Printf("originals-path %s\n", conf.OriginalsPath()) fmt.Printf("import-path %s\n", conf.ImportPath()) fmt.Printf("export-path %s\n", conf.ExportPath()) + fmt.Printf("temp-path %s\n", conf.TempPath()) fmt.Printf("cache-path %s\n", conf.CachePath()) fmt.Printf("thumbnails-path %s\n", conf.ThumbnailsPath()) fmt.Printf("resources-path %s\n", conf.ResourcesPath()) diff --git a/internal/config/filenames.go b/internal/config/filenames.go index b0509bb14..2b5a7edea 100644 --- a/internal/config/filenames.go +++ b/internal/config/filenames.go @@ -53,6 +53,10 @@ func (c *Config) CreateDirectories() error { return createError(c.ExportPath(), err) } + if err := os.MkdirAll(c.TempPath(), os.ModePerm); err != nil { + return createError(c.TempPath(), err) + } + if err := os.MkdirAll(c.ThumbnailsPath(), os.ModePerm); err != nil { return createError(c.ThumbnailsPath(), err) } @@ -156,6 +160,15 @@ func (c *Config) ExifToolBin() string { return findExecutable(c.config.ExifToolBin, "exiftool") } +// TempPath returns a temporary directory name for uploads and downloads. +func (c *Config) TempPath() string { + if c.config.TempPath == "" { + return os.TempDir() + "/photoprism" + } + + return fs.Abs(c.config.TempPath) +} + // CachePath returns the path to the cache. func (c *Config) CachePath() string { return fs.Abs(c.config.CachePath) diff --git a/internal/config/flags.go b/internal/config/flags.go index b33e82a04..b36e83fc7 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -132,6 +132,12 @@ var GlobalFlags = []cli.Flag{ Value: "~/Pictures/Export", EnvVar: "PHOTOPRISM_EXPORT_PATH", }, + cli.StringFlag{ + Name: "temp-path", + Usage: "temporary `PATH` for uploads and downloads", + Value: "", + EnvVar: "PHOTOPRISM_TEMP_PATH", + }, cli.StringFlag{ Name: "cache-path", Usage: "cache `PATH`", diff --git a/internal/config/params.go b/internal/config/params.go index a8517e553..09dff5274 100644 --- a/internal/config/params.go +++ b/internal/config/params.go @@ -47,6 +47,7 @@ type Params struct { LogLevel string `yaml:"log-level" flag:"log-level"` ConfigFile string ConfigPath string `yaml:"config-path" flag:"config-path"` + TempPath string `yaml:"temp-path" flag:"temp-path"` CachePath string `yaml:"cache-path" flag:"cache-path"` OriginalsPath string `yaml:"originals-path" flag:"originals-path"` ImportPath string `yaml:"import-path" flag:"import-path"` diff --git a/internal/remote/webdav/webdav.go b/internal/remote/webdav/webdav.go index ac6cd9048..429bf86e5 100644 --- a/internal/remote/webdav/webdav.go +++ b/internal/remote/webdav/webdav.go @@ -98,7 +98,11 @@ func (c Client) Directories(root string, recursive bool) (result fs.FileInfos, e } // Download downloads a single file to the given location. -func (c Client) Download(from, to string) error { +func (c Client) Download(from, to string, force bool) error { + if _, err := os.Stat(to); err == nil && !force { + return fmt.Errorf("webdav: download skipped, %s already exists", to) + } + dir := path.Dir(to) dirInfo, err := os.Stat(dir) @@ -121,7 +125,7 @@ func (c Client) Download(from, to string) error { } // DownloadDir downloads all files from a remote to a local directory. -func (c Client) DownloadDir(from, to string, recursive bool) (errs []error) { +func (c Client) DownloadDir(from, to string, recursive, force bool) (errs []error) { files, err := c.Files(from) if err != nil { @@ -139,7 +143,7 @@ func (c Client) DownloadDir(from, to string, recursive bool) (errs []error) { continue } - if err := c.Download(file.Abs, dest); err != nil { + if err := c.Download(file.Abs, dest, force); err != nil { msg := fmt.Errorf("webdav: %s", err) errs = append(errs, msg) log.Error(msg) @@ -154,7 +158,7 @@ func (c Client) DownloadDir(from, to string, recursive bool) (errs []error) { dirs, err := c.Directories(from, false) for _, dir := range dirs { - errs = append(errs, c.DownloadDir(dir.Abs, to, true)...) + errs = append(errs, c.DownloadDir(dir.Abs, to, true, force)...) } return errs diff --git a/internal/remote/webdav/webdav_test.go b/internal/remote/webdav/webdav_test.go index e3613c466..15074dd99 100644 --- a/internal/remote/webdav/webdav_test.go +++ b/internal/remote/webdav/webdav_test.go @@ -91,7 +91,7 @@ func TestClient_Download(t *testing.T) { t.Fatal("no files to download") } - if err := c.Download(files[0].Abs, tempFile); err != nil { + if err := c.Download(files[0].Abs, tempFile, false); err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestClient_DownloadDir(t *testing.T) { t.Run("non-recursive", func(t *testing.T) { tempDir := os.TempDir() + rnd.UUID() - if errs := c.DownloadDir("Photos", tempDir, false); len(errs) > 0 { + if errs := c.DownloadDir("Photos", tempDir, false, false); len(errs) > 0 { t.Fatal(errs) } @@ -124,7 +124,7 @@ func TestClient_DownloadDir(t *testing.T) { t.Run("recursive", func(t *testing.T) { tempDir := os.TempDir() + rnd.UUID() - if errs := c.DownloadDir("Photos", tempDir, true); len(errs) > 0 { + if errs := c.DownloadDir("Photos", tempDir, true, false); len(errs) > 0 { t.Fatal(errs) } diff --git a/internal/workers/share.go b/internal/workers/share.go index 335db42d1..9408b6967 100644 --- a/internal/workers/share.go +++ b/internal/workers/share.go @@ -42,8 +42,10 @@ func (s *Share) Start() (err error) { db := s.conf.Db() q := query.New(db) + // Find accounts for which sharing is enabled accounts, err := q.Accounts(f) + // Upload newly shared files for _, a := range accounts { if mutex.Share.Canceled() { return nil @@ -61,7 +63,7 @@ func (s *Share) Start() (err error) { } if len(files) == 0 { - // No files to upload + // No files to upload for this account continue } @@ -125,6 +127,7 @@ func (s *Share) Start() (err error) { } } + // Remove previously shared files if expired for _, a := range accounts { if mutex.Share.Canceled() { return nil @@ -142,7 +145,7 @@ func (s *Share) Start() (err error) { } if len(files) == 0 { - // No files to remove + // No files to remove for this account continue } diff --git a/internal/workers/sync.go b/internal/workers/sync.go index 607d44bf8..435ed9eb0 100644 --- a/internal/workers/sync.go +++ b/internal/workers/sync.go @@ -9,9 +9,11 @@ import ( "github.com/photoprism/photoprism/internal/event" "github.com/photoprism/photoprism/internal/form" "github.com/photoprism/photoprism/internal/mutex" + "github.com/photoprism/photoprism/internal/photoprism" "github.com/photoprism/photoprism/internal/query" "github.com/photoprism/photoprism/internal/remote" "github.com/photoprism/photoprism/internal/remote/webdav" + "github.com/photoprism/photoprism/internal/service" ) // Sync represents a sync worker. @@ -24,6 +26,11 @@ func NewSync(conf *config.Config) *Sync { return &Sync{conf: conf} } +// DownloadPath returns a temporary download path. +func (s *Sync) DownloadPath() string { + return s.conf.TempPath() + "/sync" +} + // Start starts the sync worker. func (s *Sync) Start() (err error) { if err := mutex.Sync.Start(); err != nil { @@ -40,6 +47,9 @@ func (s *Sync) Start() (err error) { db := s.conf.Db() q := query.New(db) + runImport := false + runIndex := false + accounts, err := q.Accounts(f) for _, a := range accounts { @@ -75,18 +85,28 @@ func (s *Sync) Start() (err error) { if complete, err := s.download(a); err != nil { a.AccErrors++ a.AccError = err.Error() - } else if complete && a.SyncUpload { - a.SyncStatus = entity.AccountSyncStatusUpload } else if complete { - a.SyncStatus = entity.AccountSyncStatusSynced - a.SyncDate.Time = time.Now() - a.SyncDate.Valid = true + if a.SyncFilenames { + runIndex = true + } else { + runImport = true + } + + if a.SyncUpload { + a.SyncStatus = entity.AccountSyncStatusUpload + } else { + event.Publish("sync.synced", event.Data{"account": a}) + a.SyncStatus = entity.AccountSyncStatusSynced + a.SyncDate.Time = time.Now() + a.SyncDate.Valid = true + } } case entity.AccountSyncStatusUpload: if complete, err := s.upload(a); err != nil { a.AccErrors++ a.AccError = err.Error() } else if complete { + event.Publish("sync.synced", event.Data{"account": a}) a.SyncStatus = entity.AccountSyncStatusSynced a.SyncDate.Time = time.Now() a.SyncDate.Valid = true @@ -108,6 +128,16 @@ func (s *Sync) Start() (err error) { } } + if runImport { + opt := photoprism.ImportOptionsMove(s.DownloadPath()) + service.Import().Start(opt) + } + + if runIndex { + opt := photoprism.IndexOptionsNone() + service.Index().Start(opt) + } + return err } @@ -174,7 +204,6 @@ func (s *Sync) download(a entity.Account) (complete bool, err error) { } if len(files) == 0 { - // TODO: Subscribe event to start indexing / importing event.Publish("sync.downloaded", event.Data{"account": a}) return true, nil } @@ -186,7 +215,7 @@ func (s *Sync) download(a entity.Account) (complete bool, err error) { if a.SyncFilenames { baseDir = s.conf.OriginalsPath() } else { - baseDir = fmt.Sprintf("%s/sync/%d", s.conf.ImportPath(), a.ID) + baseDir = fmt.Sprintf("%s/%d", s.DownloadPath(), a.ID) } for _, file := range files { @@ -201,10 +230,12 @@ func (s *Sync) download(a entity.Account) (complete bool, err error) { localName := baseDir + file.RemoteName - if err := client.Download(file.RemoteName, localName); err != nil { + if err := client.Download(file.RemoteName, localName, false); err != nil { + log.Errorf("sync: %s", err.Error()) file.Errors++ file.Error = err.Error() } else { + log.Infof("sync: downloaded %s from %s", file.RemoteName, a.AccName) file.Status = entity.FileSyncDownloaded } diff --git a/internal/workers/workers.go b/internal/workers/workers.go index 6dc773c16..3e4e89011 100644 --- a/internal/workers/workers.go +++ b/internal/workers/workers.go @@ -11,8 +11,9 @@ import ( var log = event.Log var stop = make(chan bool, 1) +// Start runs the service workers every 10 minutes. func Start(conf *config.Config) { - ticker := time.NewTicker(15 * time.Minute) + ticker := time.NewTicker(10 * time.Minute) go func() { for { @@ -31,6 +32,12 @@ func Start(conf *config.Config) { }() } +// Stop shuts down all service workers. +func Stop() { + stop <- true +} + +// StartShare runs the share worker once. func StartShare(conf *config.Config) { if !mutex.Share.Busy() { go func() { @@ -42,6 +49,7 @@ func StartShare(conf *config.Config) { } } +// StartShare runs the sync worker once. func StartSync(conf *config.Config) { if !mutex.Sync.Busy() { go func() { @@ -52,7 +60,3 @@ func StartSync(conf *config.Config) { }() } } - -func Stop() { - stop <- true -}