johanels/photoprism
1
0
Fork 0
mirror of https://github.com/photoprism/photoprism.git synced 2025-12-12 00:34:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

Sync: Download to temp path #225

Browse Source 
Signed-off-by: Michael Mayer <michael@liquidbytes.net>
This commit is contained in:
Michael Mayer
2020-04-06 16:34:29 +02:00
parent 9b7a5d69d4
commit 087c56e371
11 changed files with 89 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docker/photoprism-arm64/Dockerfile
View File

@@ -142,6 +142,7 @@ ENV PHOTOPRISM_ORIGINALS_PATH /photoprism/originals
ENV PHOTOPRISM_IMPORT_PATH /photoprism/import
ENV PHOTOPRISM_EXPORT_PATH /photoprism/export
ENV PHOTOPRISM_DATABASE_PATH /photoprism/database
ENV PHOTOPRISM_TEMP_PATH /photoprism/temp
ENV PHOTOPRISM_CACHE_PATH /photoprism/cache
ENV PHOTOPRISM_CONFIG_PATH /photoprism/config
ENV PHOTOPRISM_CONFIG_FILE /photoprism/config/photoprism.yml
@@ -161,6 +162,7 @@ RUN mkdir -p \
/photoprism/import \
/photoprism/export \
/photoprism/database \
/photoprism/temp \
/photoprism/cache
RUN chmod -R 777 /photoprism

2
docker/photoprism/Dockerfile
View File

@@ -58,6 +58,7 @@ ENV PHOTOPRISM_ORIGINALS_PATH /photoprism/originals
ENV PHOTOPRISM_IMPORT_PATH /photoprism/import
ENV PHOTOPRISM_EXPORT_PATH /photoprism/export
ENV PHOTOPRISM_DATABASE_PATH /photoprism/database
ENV PHOTOPRISM_TEMP_PATH /photoprism/temp
ENV PHOTOPRISM_CACHE_PATH /photoprism/cache
ENV PHOTOPRISM_CONFIG_PATH /photoprism/config
ENV PHOTOPRISM_CONFIG_FILE /photoprism/config/photoprism.yml
@@ -77,6 +78,7 @@ RUN mkdir -p \
/photoprism/import \
/photoprism/export \
/photoprism/database \
/photoprism/temp \
/photoprism/cache
RUN chmod -R 777 /photoprism

1
internal/commands/config.go
View File

@@ -45,6 +45,7 @@ func configAction(ctx *cli.Context) error {
fmt.Printf("originals-path %s\n", conf.OriginalsPath())
fmt.Printf("import-path %s\n", conf.ImportPath())
fmt.Printf("export-path %s\n", conf.ExportPath())
fmt.Printf("temp-path %s\n", conf.TempPath())
fmt.Printf("cache-path %s\n", conf.CachePath())
fmt.Printf("thumbnails-path %s\n", conf.ThumbnailsPath())
fmt.Printf("resources-path %s\n", conf.ResourcesPath())

13
internal/config/filenames.go
View File

@@ -53,6 +53,10 @@ func (c *Config) CreateDirectories() error {
return createError(c.ExportPath(), err)
}
if err := os.MkdirAll(c.TempPath(), os.ModePerm); err != nil {
return createError(c.TempPath(), err)
}
if err := os.MkdirAll(c.ThumbnailsPath(), os.ModePerm); err != nil {
return createError(c.ThumbnailsPath(), err)
}
@@ -156,6 +160,15 @@ func (c *Config) ExifToolBin() string {
return findExecutable(c.config.ExifToolBin, "exiftool")
}
// TempPath returns a temporary directory name for uploads and downloads.
func (c *Config) TempPath() string {
if c.config.TempPath == "" {
return os.TempDir() + "/photoprism"
}
return fs.Abs(c.config.TempPath)
}
// CachePath returns the path to the cache.
func (c *Config) CachePath() string {
return fs.Abs(c.config.CachePath)

6
internal/config/flags.go
View File

@@ -132,6 +132,12 @@ var GlobalFlags = []cli.Flag{
Value: "~/Pictures/Export",
EnvVar: "PHOTOPRISM_EXPORT_PATH",
},
cli.StringFlag{
Name: "temp-path",
Usage: "temporary `PATH` for uploads and downloads",
Value: "",
EnvVar: "PHOTOPRISM_TEMP_PATH",
},
cli.StringFlag{
Name: "cache-path",
Usage: "cache `PATH`",

1
internal/config/params.go
View File

@@ -47,6 +47,7 @@ type Params struct {
LogLevel string `yaml:"log-level" flag:"log-level"`
ConfigFile string
ConfigPath string `yaml:"config-path" flag:"config-path"`
TempPath string `yaml:"temp-path" flag:"temp-path"`
CachePath string `yaml:"cache-path" flag:"cache-path"`
OriginalsPath string `yaml:"originals-path" flag:"originals-path"`
ImportPath string `yaml:"import-path" flag:"import-path"`

12
internal/remote/webdav/webdav.go
View File

@@ -98,7 +98,11 @@ func (c Client) Directories(root string, recursive bool) (result fs.FileInfos, e
}
// Download downloads a single file to the given location.
func (c Client) Download(from, to string) error {
func (c Client) Download(from, to string, force bool) error {
if _, err := os.Stat(to); err == nil && !force {
return fmt.Errorf("webdav: download skipped, %s already exists", to)
}
dir := path.Dir(to)
dirInfo, err := os.Stat(dir)
@@ -121,7 +125,7 @@ func (c Client) Download(from, to string) error {
}
// DownloadDir downloads all files from a remote to a local directory.
func (c Client) DownloadDir(from, to string, recursive bool) (errs []error) {
func (c Client) DownloadDir(from, to string, recursive, force bool) (errs []error) {
files, err := c.Files(from)
if err != nil {
@@ -139,7 +143,7 @@ func (c Client) DownloadDir(from, to string, recursive bool) (errs []error) {
continue
}
if err := c.Download(file.Abs, dest); err != nil {
if err := c.Download(file.Abs, dest, force); err != nil {
msg := fmt.Errorf("webdav: %s", err)
errs = append(errs, msg)
log.Error(msg)
@@ -154,7 +158,7 @@ func (c Client) DownloadDir(from, to string, recursive bool) (errs []error) {
dirs, err := c.Directories(from, false)
for _, dir := range dirs {
errs = append(errs, c.DownloadDir(dir.Abs, to, true)...)
errs = append(errs, c.DownloadDir(dir.Abs, to, true, force)...)
}
return errs

6
internal/remote/webdav/webdav_test.go
View File

@@ -91,7 +91,7 @@ func TestClient_Download(t *testing.T) {
t.Fatal("no files to download")
}
if err := c.Download(files[0].Abs, tempFile); err != nil {
if err := c.Download(files[0].Abs, tempFile, false); err != nil {
t.Fatal(err)
}
@@ -112,7 +112,7 @@ func TestClient_DownloadDir(t *testing.T) {
t.Run("non-recursive", func(t *testing.T) {
tempDir := os.TempDir() + rnd.UUID()
if errs := c.DownloadDir("Photos", tempDir, false); len(errs) > 0 {
if errs := c.DownloadDir("Photos", tempDir, false, false); len(errs) > 0 {
t.Fatal(errs)
}
@@ -124,7 +124,7 @@ func TestClient_DownloadDir(t *testing.T) {
t.Run("recursive", func(t *testing.T) {
tempDir := os.TempDir() + rnd.UUID()
if errs := c.DownloadDir("Photos", tempDir, true); len(errs) > 0 {
if errs := c.DownloadDir("Photos", tempDir, true, false); len(errs) > 0 {
t.Fatal(errs)
}

7
internal/workers/share.go
View File

@@ -42,8 +42,10 @@ func (s *Share) Start() (err error) {
db := s.conf.Db()
q := query.New(db)
// Find accounts for which sharing is enabled
accounts, err := q.Accounts(f)
// Upload newly shared files
for _, a := range accounts {
if mutex.Share.Canceled() {
return nil
@@ -61,7 +63,7 @@ func (s *Share) Start() (err error) {
}
if len(files) == 0 {
// No files to upload
// No files to upload for this account
continue
}
@@ -125,6 +127,7 @@ func (s *Share) Start() (err error) {
}
}
// Remove previously shared files if expired
for _, a := range accounts {
if mutex.Share.Canceled() {
return nil
@@ -142,7 +145,7 @@ func (s *Share) Start() (err error) {
}
if len(files) == 0 {
// No files to remove
// No files to remove for this account
continue
}

47
internal/workers/sync.go
View File

@@ -9,9 +9,11 @@ import (
"github.com/photoprism/photoprism/internal/event"
"github.com/photoprism/photoprism/internal/form"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/photoprism"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/internal/remote"
"github.com/photoprism/photoprism/internal/remote/webdav"
"github.com/photoprism/photoprism/internal/service"
)
// Sync represents a sync worker.
@@ -24,6 +26,11 @@ func NewSync(conf *config.Config) *Sync {
return &Sync{conf: conf}
}
// DownloadPath returns a temporary download path.
func (s *Sync) DownloadPath() string {
return s.conf.TempPath() + "/sync"
}
// Start starts the sync worker.
func (s *Sync) Start() (err error) {
if err := mutex.Sync.Start(); err != nil {
@@ -40,6 +47,9 @@ func (s *Sync) Start() (err error) {
db := s.conf.Db()
q := query.New(db)
runImport := false
runIndex := false
accounts, err := q.Accounts(f)
for _, a := range accounts {
@@ -75,18 +85,28 @@ func (s *Sync) Start() (err error) {
if complete, err := s.download(a); err != nil {
a.AccErrors++
a.AccError = err.Error()
} else if complete && a.SyncUpload {
a.SyncStatus = entity.AccountSyncStatusUpload
} else if complete {
a.SyncStatus = entity.AccountSyncStatusSynced
a.SyncDate.Time = time.Now()
a.SyncDate.Valid = true
if a.SyncFilenames {
runIndex = true
} else {
runImport = true
}
if a.SyncUpload {
a.SyncStatus = entity.AccountSyncStatusUpload
} else {
event.Publish("sync.synced", event.Data{"account": a})
a.SyncStatus = entity.AccountSyncStatusSynced
a.SyncDate.Time = time.Now()
a.SyncDate.Valid = true
}
}
case entity.AccountSyncStatusUpload:
if complete, err := s.upload(a); err != nil {
a.AccErrors++
a.AccError = err.Error()
} else if complete {
event.Publish("sync.synced", event.Data{"account": a})
a.SyncStatus = entity.AccountSyncStatusSynced
a.SyncDate.Time = time.Now()
a.SyncDate.Valid = true
@@ -108,6 +128,16 @@ func (s *Sync) Start() (err error) {
}
}
if runImport {
opt := photoprism.ImportOptionsMove(s.DownloadPath())
service.Import().Start(opt)
}
if runIndex {
opt := photoprism.IndexOptionsNone()
service.Index().Start(opt)
}
return err
}
@@ -174,7 +204,6 @@ func (s *Sync) download(a entity.Account) (complete bool, err error) {
}
if len(files) == 0 {
// TODO: Subscribe event to start indexing / importing
event.Publish("sync.downloaded", event.Data{"account": a})
return true, nil
}
@@ -186,7 +215,7 @@ func (s *Sync) download(a entity.Account) (complete bool, err error) {
if a.SyncFilenames {
baseDir = s.conf.OriginalsPath()
} else {
baseDir = fmt.Sprintf("%s/sync/%d", s.conf.ImportPath(), a.ID)
baseDir = fmt.Sprintf("%s/%d", s.DownloadPath(), a.ID)
}
for _, file := range files {
@@ -201,10 +230,12 @@ func (s *Sync) download(a entity.Account) (complete bool, err error) {
localName := baseDir + file.RemoteName
if err := client.Download(file.RemoteName, localName); err != nil {
if err := client.Download(file.RemoteName, localName, false); err != nil {
log.Errorf("sync: %s", err.Error())
file.Errors++
file.Error = err.Error()
} else {
log.Infof("sync: downloaded %s from %s", file.RemoteName, a.AccName)
file.Status = entity.FileSyncDownloaded
}

14
internal/workers/workers.go
View File

@@ -11,8 +11,9 @@ import (
var log = event.Log
var stop = make(chan bool, 1)
// Start runs the service workers every 10 minutes.
func Start(conf *config.Config) {
ticker := time.NewTicker(15 * time.Minute)
ticker := time.NewTicker(10 * time.Minute)
go func() {
for {
@@ -31,6 +32,12 @@ func Start(conf *config.Config) {
}()
}
// Stop shuts down all service workers.
func Stop() {
stop <- true
}
// StartShare runs the share worker once.
func StartShare(conf *config.Config) {
if !mutex.Share.Busy() {
go func() {
@@ -42,6 +49,7 @@ func StartShare(conf *config.Config) {
}
}
// StartShare runs the sync worker once.
func StartSync(conf *config.Config) {
if !mutex.Sync.Busy() {
go func() {
@@ -52,7 +60,3 @@ func StartSync(conf *config.Config) {
}()
}
}
func Stop() {
stop <- true
}