Backups: Refactor config and add "sidecar-yaml" config option #4243

Signed-off-by: Michael Mayer <michael@photoprism.app>
This commit is contained in:
Michael Mayer
2024-05-13 09:21:34 +02:00
parent e042b40975
commit 0396e86f4e
31 changed files with 407 additions and 226 deletions

View File

@@ -36,6 +36,7 @@ export class ConfigOptions extends Model {
OriginalsLimit: 0,
Workers: 0,
WakeupInterval: 0,
BackupIndex: false,
DisableWebDAV: config.values.disable.webdav,
DisableSettings: config.values.disable.settings,
DisablePlaces: config.values.disable.places,

View File

@@ -19,7 +19,7 @@
<v-checkbox
v-model="settings.ReadOnly"
:disabled="busy"
class="ma-0 pa-0 input-private"
class="ma-0 pa-0 input-readonly"
color="secondary-dark"
:label="$gettext('Read-Only Mode')"
:hint="$gettext('Don\'t modify originals folder. Disables import, upload, and delete.')"
@@ -37,12 +37,12 @@
<v-flex xs12 sm6 lg3 class="px-2 pb-2 pt-2">
<v-checkbox
v-model="settings.DisableBackups"
v-model="settings.BackupIndex"
:disabled="busy"
class="ma-0 pa-0 input-private"
class="ma-0 pa-0 input-backup-index"
color="secondary-dark"
:label="$gettext('Disable Backups')"
:hint="$gettext('Don\'t backup photo and album metadata to YAML files.')"
:label="$gettext('Database Backups')"
:hint="$gettext('Create index backups based on the configured schedule.')"
prepend-icon="healing"
persistent-hint
@change="onChange"
@@ -54,7 +54,7 @@
<v-checkbox
v-model="settings.DisableWebDAV"
:disabled="busy"
class="ma-0 pa-0 input-private"
class="ma-0 pa-0 input-disable-webdav"
color="secondary-dark"
:label="$gettext('Disable WebDAV')"
:hint="$gettext('Disable built-in WebDAV server. Requires a restart.')"
@@ -74,7 +74,7 @@
<v-checkbox
v-model="settings.DisableExifTool"
:disabled="busy"
class="ma-0 pa-0 input-private"
class="ma-0 pa-0 input-disable-exiftool"
color="secondary-dark"
:label="$gettext('Disable ExifTool')"
:hint="$gettext('Don\'t create ExifTool JSON files for improved metadata extraction.')"
@@ -89,7 +89,7 @@
<v-checkbox
v-model="settings.DisableTensorFlow"
:disabled="busy"
class="ma-0 pa-0 input-private"
class="ma-0 pa-0 input-disable-tensorflow"
color="secondary-dark"
:label="$gettext('Disable TensorFlow')"
:hint="$gettext('Don\'t use TensorFlow for image classification.')"

View File

@@ -20,16 +20,17 @@ import (
var albumMutex = sync.Mutex{}
// SaveAlbumAsYaml saves album data as YAML file.
func SaveAlbumAsYaml(a entity.Album) {
// SaveAlbumYaml backs up the album metadata in YAML files.
func SaveAlbumYaml(a entity.Album) {
c := get.Config()
// Write YAML sidecar file (optional).
if !c.BackupYaml() {
// Check if album YAML backups are enabled.
if !c.BackupAlbums() {
return
}
fileName := a.YamlFileName(c.AlbumsPath())
// Create or update album backup file.
fileName := a.YamlFileName(c.BackupAlbumsPath())
if err := a.SaveAsYaml(fileName); err != nil {
log.Errorf("album: %s (update yaml)", err)
@@ -121,7 +122,7 @@ func CreateAlbum(router *gin.RouterGroup) {
UpdateClientConfig()
// Update album YAML backup.
SaveAlbumAsYaml(*a)
SaveAlbumYaml(*a)
// Return as JSON.
c.JSON(http.StatusOK, a)
@@ -187,7 +188,7 @@ func UpdateAlbum(router *gin.RouterGroup) {
UpdateClientConfig()
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
c.JSON(http.StatusOK, a)
})
@@ -244,7 +245,7 @@ func DeleteAlbum(router *gin.RouterGroup) {
UpdateClientConfig()
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
c.JSON(http.StatusOK, a)
})
@@ -292,7 +293,7 @@ func LikeAlbum(router *gin.RouterGroup) {
PublishAlbumEvent(StatusUpdated, uid, c)
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
c.JSON(http.StatusOK, i18n.NewResponse(http.StatusOK, i18n.MsgChangesSaved))
})
@@ -340,7 +341,7 @@ func DislikeAlbum(router *gin.RouterGroup) {
PublishAlbumEvent(StatusUpdated, uid, c)
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
c.JSON(http.StatusOK, i18n.NewResponse(http.StatusOK, i18n.MsgChangesSaved))
})
@@ -408,7 +409,7 @@ func CloneAlbums(router *gin.RouterGroup) {
PublishAlbumEvent(StatusUpdated, a.AlbumUID, c)
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
}
c.JSON(http.StatusOK, gin.H{"code": http.StatusOK, "message": i18n.Msg(i18n.MsgAlbumCloned), "album": a, "added": added})
@@ -482,7 +483,7 @@ func AddPhotosToAlbum(router *gin.RouterGroup) {
PublishAlbumEvent(StatusUpdated, a.AlbumUID, c)
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
// Auto-approve photos that have been added to an album,
// see https://github.com/photoprism/photoprism/issues/4229
@@ -500,7 +501,7 @@ func AddPhotosToAlbum(router *gin.RouterGroup) {
log.Errorf("approve: %s", err)
} else {
approved = append(approved, p)
SavePhotoAsYaml(&p)
SaveSidecarYaml(&p)
}
}
@@ -575,7 +576,7 @@ func RemovePhotosFromAlbum(router *gin.RouterGroup) {
PublishAlbumEvent(StatusUpdated, a.AlbumUID, c)
// Update album YAML backup.
SaveAlbumAsYaml(a)
SaveAlbumYaml(a)
}
c.JSON(http.StatusOK, gin.H{"code": http.StatusOK, "message": i18n.Msg(i18n.MsgChangesSaved), "album": a, "photos": f.Photos, "removed": removed})

View File

@@ -47,7 +47,7 @@ func BatchPhotosArchive(router *gin.RouterGroup) {
log.Infof("photos: archiving %s", clean.Log(f.String()))
if get.Config().BackupYaml() {
if get.Config().SidecarYaml() {
// Fetch selection from index.
photos, err := query.SelectedPhotos(f)
@@ -60,7 +60,7 @@ func BatchPhotosArchive(router *gin.RouterGroup) {
if archiveErr := p.Archive(); archiveErr != nil {
log.Errorf("archive: %s", archiveErr)
} else {
SavePhotoAsYaml(&p)
SaveSidecarYaml(&p)
}
}
} else if err := entity.Db().Where("photo_uid IN (?)", f.Photos).Delete(&entity.Photo{}).Error; err != nil {
@@ -110,7 +110,7 @@ func BatchPhotosRestore(router *gin.RouterGroup) {
log.Infof("photos: restoring %s", clean.Log(f.String()))
if get.Config().BackupYaml() {
if get.Config().SidecarYaml() {
// Fetch selection from index.
photos, err := query.SelectedPhotos(f)
@@ -123,7 +123,7 @@ func BatchPhotosRestore(router *gin.RouterGroup) {
if err = p.Restore(); err != nil {
log.Errorf("restore: %s", err)
} else {
SavePhotoAsYaml(&p)
SaveSidecarYaml(&p)
}
}
} else if err := entity.Db().Unscoped().Model(&entity.Photo{}).Where("photo_uid IN (?)", f.Photos).
@@ -187,7 +187,7 @@ func BatchPhotosApprove(router *gin.RouterGroup) {
log.Errorf("approve: %s", err)
} else {
approved = append(approved, p)
SavePhotoAsYaml(&p)
SaveSidecarYaml(&p)
}
}
@@ -278,7 +278,7 @@ func BatchPhotosPrivate(router *gin.RouterGroup) {
// Fetch selection from index.
if photos, err := query.SelectedPhotos(f); err == nil {
for _, p := range photos {
SavePhotoAsYaml(&p)
SaveSidecarYaml(&p)
}
event.EntitiesUpdated("photos", photos)

View File

@@ -18,8 +18,8 @@ import (
"github.com/photoprism/photoprism/pkg/i18n"
)
// SavePhotoAsYaml saves photo data as YAML file.
func SavePhotoAsYaml(p *entity.Photo) {
// SaveSidecarYaml saves picture metadata as YAML file.
func SaveSidecarYaml(p *entity.Photo) {
if p == nil {
log.Debugf("api: photo is nil (update yaml)")
return
@@ -27,15 +27,16 @@ func SavePhotoAsYaml(p *entity.Photo) {
c := get.Config()
// Write YAML sidecar file (optional).
if !c.BackupYaml() {
// Check if creating and updating YAML sidecar files is enabled.
if !c.SidecarYaml() {
return
}
// Create or update metadata export in YAML sidecar file.
fileName := p.YamlFileName(c.OriginalsPath(), c.SidecarPath())
if err := p.SaveAsYaml(fileName); err != nil {
log.Errorf("photo: %s (update yaml)", err)
log.Errorf("photo: %s (save yaml)", err)
} else {
log.Debugf("photo: updated yaml file %s", clean.Log(filepath.Base(fileName)))
}
@@ -119,7 +120,7 @@ func UpdatePhoto(router *gin.RouterGroup) {
return
}
SavePhotoAsYaml(&p)
SaveSidecarYaml(&p)
UpdateClientConfig()
@@ -230,7 +231,7 @@ func ApprovePhoto(router *gin.RouterGroup) {
return
}
SavePhotoAsYaml(&m)
SaveSidecarYaml(&m)
PublishPhotoEvent(StatusUpdated, id, c)

View File

@@ -44,7 +44,7 @@ func LikePhoto(router *gin.RouterGroup) {
return
}
SavePhotoAsYaml(&m)
SaveSidecarYaml(&m)
PublishPhotoEvent(StatusUpdated, id, c)
}
@@ -84,7 +84,7 @@ func DislikePhoto(router *gin.RouterGroup) {
return
}
SavePhotoAsYaml(&m)
SaveSidecarYaml(&m)
PublishPhotoEvent(StatusUpdated, id, c)
}

View File

@@ -37,19 +37,19 @@ var backupFlags = []cli.Flag{
},
cli.BoolFlag{
Name: "albums, a",
Usage: "create album YAML file backups in the configured backup path",
Usage: "write album metadata to YAML files",
},
cli.StringFlag{
Name: "albums-path",
Usage: "custom `PATH` for creating album backups",
Usage: "custom album backup `PATH`",
},
cli.BoolFlag{
Name: "index, i",
Usage: "create index backup in the configured backup path (stdout if - is passed as first argument)",
Usage: "create index database backup (sent to stdout if - is passed as first argument)",
},
cli.StringFlag{
Name: "index-path",
Usage: "custom `PATH` for creating index backups",
Usage: "custom index backup `PATH`",
},
cli.IntFlag{
Name: "retain, r",
@@ -95,7 +95,7 @@ func backupAction(ctx *cli.Context) error {
log.Warnf("custom index backup path not writable, using default")
}
backupPath = filepath.Join(conf.BackupPath(), conf.DatabaseDriver())
backupPath = conf.BackupIndexPath()
}
backupFile := time.Now().UTC().Format("2006-01-02") + ".sql"
@@ -113,10 +113,10 @@ func backupAction(ctx *cli.Context) error {
log.Warnf("album files path not writable, using default")
}
albumsPath = conf.AlbumsPath()
albumsPath = conf.BackupAlbumsPath()
}
log.Infof("saving albums in %s", clean.Log(albumsPath))
log.Infof("creating album YAML files in %s", clean.Log(albumsPath))
if count, backupErr := photoprism.BackupAlbums(albumsPath, true); backupErr != nil {
return backupErr

View File

@@ -137,7 +137,7 @@ func resetAction(ctx *cli.Context) error {
if _, err := removeAlbumYamlPrompt.Run(); err == nil {
start := time.Now()
matches, err := filepath.Glob(regexp.QuoteMeta(conf.AlbumsPath()) + "/**/*.yml")
matches, err := filepath.Glob(regexp.QuoteMeta(conf.BackupAlbumsPath()) + "/**/*.yml")
if err != nil {
return err

View File

@@ -34,19 +34,19 @@ var restoreFlags = []cli.Flag{
},
cli.BoolFlag{
Name: "albums, a",
Usage: "restore album YAML file backups from the configured backup path",
Usage: "restore album metadata from YAML backup files",
},
cli.StringFlag{
Name: "albums-path",
Usage: "custom `PATH` for restoring album backups",
Usage: "custom album backup `PATH`",
},
cli.BoolFlag{
Name: "index, i",
Usage: "restore index from the latest backup in the configured backup path (or the file passed as first argument)",
Usage: "restore index from the specified file or the most recent file in the backup path (from stdin if - is passed as first argument)",
},
cli.StringFlag{
Name: "index-path",
Usage: "custom `PATH` for restoring index backups",
Usage: "custom index backup `PATH`",
},
}
@@ -93,7 +93,7 @@ func restoreAction(ctx *cli.Context) error {
get.SetConfig(conf)
if albumsPath == "" {
albumsPath = conf.AlbumsPath()
albumsPath = conf.BackupAlbumsPath()
}
if !fs.PathExists(albumsPath) {

View File

@@ -125,7 +125,7 @@ func startAction(ctx *cli.Context) error {
go server.Start(cctx, conf)
// Restore albums from YAML files.
if count, restoreErr := photoprism.RestoreAlbums(conf.AlbumsPath(), false); restoreErr != nil {
if count, restoreErr := photoprism.RestoreAlbums(conf.BackupAlbumsPath(), false); restoreErr != nil {
log.Errorf("restore: %s", restoreErr)
} else if count > 0 {
log.Infof("%d albums restored", count)

View File

@@ -330,7 +330,7 @@ func (c *Config) Init() error {
// readSerial reads and returns the current storage serial.
func (c *Config) readSerial() string {
storageName := filepath.Join(c.StoragePath(), serialName)
backupName := filepath.Join(c.BackupPath(), serialName)
backupName := c.BackupPath(serialName)
if fs.FileExists(storageName) {
if data, err := os.ReadFile(storageName); err == nil && len(data) == 16 {
@@ -360,7 +360,7 @@ func (c *Config) InitSerial() (err error) {
c.serial = rnd.GenerateUID('z')
storageName := filepath.Join(c.StoragePath(), serialName)
backupName := filepath.Join(c.BackupPath(), serialName)
backupName := c.BackupPath(serialName)
if err = os.WriteFile(storageName, []byte(c.serial), fs.ModeFile); err != nil {
return fmt.Errorf("could not create %s: %s", storageName, err)

View File

@@ -5,6 +5,7 @@ import (
"github.com/robfig/cron/v3"
"github.com/photoprism/photoprism/pkg/clean"
"github.com/photoprism/photoprism/pkg/fs"
)
@@ -13,8 +14,17 @@ const (
DefaultBackupRetain = 14
)
// BackupPath returns the backup storage path.
func (c *Config) BackupPath() string {
// BackupPath returns the backup storage path based on the specified type, or the base path if none is specified.
func (c *Config) BackupPath(backupType string) string {
if s := clean.TypeLowerUnderscore(backupType); s == "" {
return c.BackupBasePath()
} else {
return filepath.Join(c.BackupBasePath(), s)
}
}
// BackupBasePath returns the backup storage base path.
func (c *Config) BackupBasePath() string {
if fs.PathWritable(c.options.BackupPath) {
return fs.Abs(c.options.BackupPath)
}
@@ -22,25 +32,22 @@ func (c *Config) BackupPath() string {
return filepath.Join(c.StoragePath(), "backup")
}
// BackupIndex checks if SQL database dumps should be created based on the configured schedule.
func (c *Config) BackupIndex() bool {
return c.options.BackupIndex
}
// BackupAlbums checks if album YAML file backups should be created based on the configured schedule.
func (c *Config) BackupAlbums() bool {
return c.options.BackupAlbums
}
// BackupRetain returns the maximum number of SQL database dumps to keep, or -1 to keep all.
func (c *Config) BackupRetain() int {
if c.options.BackupRetain == 0 {
return DefaultBackupRetain
} else if c.options.BackupRetain < -1 {
return -1
// BackupAlbumsPath returns the backup path for album YAML files.
func (c *Config) BackupAlbumsPath() string {
if dir := filepath.Join(c.StoragePath(), "albums"); fs.PathExists(dir) {
return dir
}
return c.options.BackupRetain
return c.BackupPath("albums")
}
// BackupIndexPath returns the backup path for index database dumps.
func (c *Config) BackupIndexPath() string {
if driver := c.DatabaseDriver(); driver != "" {
return c.BackupPath(driver)
}
return c.BackupPath("index")
}
// BackupSchedule returns the backup schedule in cron format, e.g. "0 12 * * *" for daily at noon.
@@ -54,3 +61,33 @@ func (c *Config) BackupSchedule() string {
return c.options.BackupSchedule
}
// BackupRetain returns the maximum number of SQL database dumps to keep, or -1 to keep all.
func (c *Config) BackupRetain() int {
if c.options.BackupRetain == 0 {
return DefaultBackupRetain
} else if c.options.BackupRetain < -1 {
return -1
}
return c.options.BackupRetain
}
// BackupIndex checks if SQL database dumps should be created based on the configured schedule.
func (c *Config) BackupIndex() bool {
return c.options.BackupIndex
}
// BackupAlbums checks if album YAML file backups should be created based on the configured schedule.
func (c *Config) BackupAlbums() bool {
return c.options.BackupAlbums
}
// DisableBackups checks if creating and updating sidecar YAML files should be disabled.
func (c *Config) DisableBackups() bool {
if !c.SidecarWritable() {
return true
}
return c.options.DisableBackups
}

View File

@@ -8,17 +8,27 @@ import (
func TestConfig_BackupPath(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Contains(t, c.BackupPath(), "/storage/testdata/backup")
assert.Contains(t, c.BackupPath(""), "/storage/testdata/backup")
}
func TestConfig_BackupIndex(t *testing.T) {
func TestConfig_BackupBasePath(t *testing.T) {
c := NewConfig(CliTestContext())
assert.False(t, c.BackupIndex())
assert.Contains(t, c.BackupBasePath(), "/storage/testdata/backup")
}
func TestConfig_BackupAlbums(t *testing.T) {
func TestConfig_BackupAlbumsPath(t *testing.T) {
c := NewConfig(CliTestContext())
assert.False(t, c.BackupAlbums())
assert.Contains(t, c.BackupAlbumsPath(), "/albums")
}
func TestConfig_BackupIndexPath(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Contains(t, c.BackupIndexPath(), "/storage/testdata/backup/sqlite")
}
func TestConfig_BackupSchedule(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, DefaultBackupSchedule, c.BackupSchedule())
}
func TestConfig_BackupRetain(t *testing.T) {
@@ -26,7 +36,30 @@ func TestConfig_BackupRetain(t *testing.T) {
assert.Equal(t, DefaultBackupRetain, c.BackupRetain())
}
func TestConfig_BackupSchedule(t *testing.T) {
func TestConfig_BackupIndex(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, DefaultBackupSchedule, c.BackupSchedule())
assert.False(t, c.BackupIndex())
c.options.BackupIndex = true
assert.True(t, c.BackupIndex())
c.options.BackupIndex = false
assert.False(t, c.BackupIndex())
}
func TestConfig_BackupAlbums(t *testing.T) {
c := NewConfig(CliTestContext())
assert.True(t, c.BackupAlbums())
c.options.BackupAlbums = false
assert.False(t, c.BackupAlbums())
c.options.BackupAlbums = true
assert.True(t, c.BackupAlbums())
}
func TestConfig_DisableBackups(t *testing.T) {
c := NewConfig(CliTestContext())
assert.False(t, c.DisableBackups())
c.options.DisableBackups = true
assert.True(t, c.DisableBackups())
c.options.DisableBackups = false
assert.False(t, c.DisableBackups())
}

View File

@@ -12,15 +12,6 @@ func (c *Config) DisableRestart() bool {
return c.options.DisableRestart
}
// DisableBackups checks if photo and album metadata files should be disabled.
func (c *Config) DisableBackups() bool {
if !c.SidecarWritable() {
return true
}
return c.options.DisableBackups
}
// DisableWebDAV checks if the built-in WebDAV server should be disabled.
func (c *Config) DisableWebDAV() bool {
if c.Public() || c.Demo() {

View File

@@ -6,11 +6,6 @@ import (
"github.com/stretchr/testify/assert"
)
func TestConfig_DisableBackups(t *testing.T) {
c := NewConfig(CliTestContext())
assert.False(t, c.DisableBackups())
}
func TestConfig_DisableWebDAV(t *testing.T) {
c := NewConfig(CliTestContext())

View File

@@ -14,8 +14,3 @@ func (c *Config) ExifToolBin() string {
func (c *Config) ExifToolJson() bool {
return !c.DisableExifTool()
}
// BackupYaml checks if creating YAML files is enabled.
func (c *Config) BackupYaml() bool {
return !c.DisableBackups()
}

View File

@@ -29,15 +29,3 @@ func TestConfig_ExifToolJson(t *testing.T) {
assert.Equal(t, false, c.ExifToolJson())
assert.Equal(t, c.DisableExifTool(), !c.ExifToolJson())
}
func TestConfig_SidecarYaml(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, true, c.BackupYaml())
assert.Equal(t, c.DisableBackups(), !c.BackupYaml())
c.options.DisableBackups = true
assert.Equal(t, false, c.BackupYaml())
assert.Equal(t, c.DisableBackups(), !c.BackupYaml())
}

View File

@@ -116,8 +116,8 @@ func (c *Config) CreateDirectories() error {
return createError(dir, err)
}
// Create backup storage path if it doesn't exist yet.
if dir := c.BackupPath(); dir == "" {
// Create backup base path if it doesn't exist yet.
if dir := c.BackupBasePath(); dir == "" {
return notFoundError("backup")
} else if err := fs.MkdirAll(dir); err != nil {
return createError(dir, err)
@@ -177,7 +177,7 @@ func (c *Config) CreateDirectories() error {
}
// Create albums backup path if it doesn't exist yet.
if dir := c.AlbumsPath(); dir == "" {
if dir := c.BackupAlbumsPath(); dir == "" {
return notFoundError("albums")
} else if err := fs.MkdirAll(dir); err != nil {
return createError(dir, err)
@@ -344,6 +344,15 @@ func (c *Config) SidecarWritable() bool {
return !c.ReadOnly() || c.SidecarPathIsAbs()
}
// SidecarYaml checks if sidecar YAML files should be created and updated.
func (c *Config) SidecarYaml() bool {
if !c.SidecarWritable() || c.options.DisableBackups {
return false
}
return c.options.SidecarYaml
}
// UsersPath returns the relative base path for user assets.
func (c *Config) UsersPath() string {
// Set default.
@@ -639,11 +648,6 @@ func (c *Config) SqliteBin() string {
return findBin("", "sqlite3")
}
// AlbumsPath returns the storage path for album YAML files.
func (c *Config) AlbumsPath() string {
return filepath.Join(c.StoragePath(), "albums")
}
// OriginalsAlbumsPath returns the optional album YAML file path inside originals.
func (c *Config) OriginalsAlbumsPath() string {
return filepath.Join(c.OriginalsPath(), "albums")

View File

@@ -23,6 +23,27 @@ func TestConfig_SidecarPath(t *testing.T) {
assert.Equal(t, "/go/src/github.com/photoprism/photoprism/storage/testdata/sidecar", c.SidecarPath())
}
func TestConfig_SidecarYaml(t *testing.T) {
c := NewConfig(NewTestContext(nil))
// t.Logf("c.options.DisableBackups = %t", c.options.DisableBackups)
// t.Logf("c.options.SidecarYaml = %t", c.options.SidecarYaml)
assert.Equal(t, true, c.SidecarYaml())
assert.Equal(t, c.DisableBackups(), !c.SidecarYaml())
c.options.DisableBackups = true
assert.Equal(t, false, c.SidecarYaml())
assert.Equal(t, c.DisableBackups(), !c.SidecarYaml())
c.options.DisableBackups = false
c.options.SidecarYaml = true
assert.Equal(t, true, c.SidecarYaml())
assert.Equal(t, c.DisableBackups(), !c.SidecarYaml())
}
func TestConfig_UsersPath(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Contains(t, c.UsersPath(), "users")
@@ -175,7 +196,7 @@ func TestConfig_TestdataPath(t *testing.T) {
func TestConfig_AlbumsPath(t *testing.T) {
c := NewConfig(CliTestContext())
assert.Equal(t, "/go/src/github.com/photoprism/photoprism/storage/testdata/albums", c.AlbumsPath())
assert.Equal(t, "/go/src/github.com/photoprism/photoprism/storage/testdata/albums", c.BackupAlbumsPath())
}
func TestConfig_OriginalsAlbumsPath(t *testing.T) {

View File

@@ -156,6 +156,11 @@ var Flags = CliFlags{
Usage: "custom relative or absolute sidecar `PATH`*optional*",
EnvVar: EnvVar("SIDECAR_PATH"),
}}, {
Flag: cli.BoolFlag{
Name: "sidecar-yaml",
Usage: "write picture metadata to YAML sidecar files",
EnvVar: EnvVar("SIDECAR_YAML"),
}}, {
Flag: cli.StringFlag{
Name: "cache-path, ca",
Usage: "custom cache `PATH` for sessions and thumbnail files*optional*",
@@ -183,18 +188,14 @@ var Flags = CliFlags{
}}, {
Flag: cli.StringFlag{
Name: "backup-path, ba",
Usage: "custom default `PATH` for creating and restoring index backups*optional*",
Usage: "custom base `PATH` for creating and restoring backups*optional*",
EnvVar: EnvVar("BACKUP_PATH"),
}}, {
Flag: cli.BoolFlag{
Name: "backup-index",
Usage: "create index backups based on the configured schedule",
EnvVar: EnvVar("BACKUP_INDEX"),
}}, {
Flag: cli.BoolFlag{
Name: "backup-albums",
Usage: "create album YAML file backups based on the configured schedule",
EnvVar: EnvVar("BACKUP_ALBUMS"),
Flag: cli.StringFlag{
Name: "backup-schedule",
Usage: "backup `SCHEDULE` in cron format, e.g. \"0 12 * * *\" for daily at noon",
Value: DefaultBackupSchedule,
EnvVar: EnvVar("BACKUP_SCHEDULE"),
}}, {
Flag: cli.IntFlag{
Name: "backup-retain",
@@ -202,11 +203,15 @@ var Flags = CliFlags{
Value: DefaultBackupRetain,
EnvVar: EnvVar("BACKUP_RETAIN"),
}}, {
Flag: cli.StringFlag{
Name: "backup-schedule",
Usage: "backup `SCHEDULE` in cron format, e.g. \"0 12 * * *\" for daily at noon",
Value: DefaultBackupSchedule,
EnvVar: EnvVar("BACKUP_SCHEDULE"),
Flag: cli.BoolFlag{
Name: "backup-index",
Usage: "create index database backups based on the configured schedule",
EnvVar: EnvVar("BACKUP_INDEX"),
}}, {
Flag: cli.BoolFlag{
Name: "backup-albums",
Usage: "write album metadata to YAML files in the backup path",
EnvVar: EnvVar("BACKUP_ALBUMS"),
}}, {
Flag: cli.IntFlag{
Name: "index-workers, workers",
@@ -260,8 +265,9 @@ var Flags = CliFlags{
}}, {
Flag: cli.BoolFlag{
Name: "disable-backups",
Usage: "disable backing up albums and photo metadata to YAML files",
Usage: "disable creating and updating YAML sidecar files (does not affect index and album backups)",
EnvVar: EnvVar("DISABLE_BACKUPS"),
Hidden: true,
}}, {
Flag: cli.BoolFlag{
Name: "disable-webdav",
@@ -345,7 +351,7 @@ var Flags = CliFlags{
}}, {
Flag: cli.BoolFlag{
Name: "detect-nsfw",
Usage: "automatically flag photos as private that MAY be offensive (requires TensorFlow)",
Usage: "automatically flag pictures as private that MAY be offensive (requires TensorFlow)",
EnvVar: EnvVar("DETECT_NSFW"),
}}, {
Flag: cli.BoolFlag{

View File

@@ -50,6 +50,7 @@ type Options struct {
UsersPath string `yaml:"UsersPath" json:"-" flag:"users-path"`
StoragePath string `yaml:"StoragePath" json:"-" flag:"storage-path"`
SidecarPath string `yaml:"SidecarPath" json:"-" flag:"sidecar-path"`
SidecarYaml bool `yaml:"SidecarYaml" json:"SidecarYaml" flag:"sidecar-yaml"`
CachePath string `yaml:"CachePath" json:"-" flag:"cache-path"`
ImportPath string `yaml:"ImportPath" json:"-" flag:"import-path"`
ImportDest string `yaml:"ImportDest" json:"-" flag:"import-dest"`
@@ -57,10 +58,10 @@ type Options struct {
CustomAssetsPath string `yaml:"-" json:"-" flag:"custom-assets-path"`
TempPath string `yaml:"TempPath" json:"-" flag:"temp-path"`
BackupPath string `yaml:"BackupPath" json:"-" flag:"backup-path"`
BackupSchedule string `yaml:"BackupSchedule" json:"BackupSchedule" flag:"backup-schedule"`
BackupRetain int `yaml:"BackupRetain" json:"BackupRetain" flag:"backup-retain"`
BackupIndex bool `yaml:"BackupIndex" json:"BackupIndex" flag:"backup-index"`
BackupAlbums bool `yaml:"BackupAlbums" json:"BackupAlbums" flag:"backup-albums"`
BackupRetain int `yaml:"BackupRetain" json:"BackupRetain" flag:"backup-retain"`
BackupSchedule string `yaml:"BackupSchedule" json:"BackupSchedule" flag:"backup-schedule"`
IndexWorkers int `yaml:"IndexWorkers" json:"IndexWorkers" flag:"index-workers"`
IndexSchedule string `yaml:"IndexSchedule" json:"IndexSchedule" flag:"index-schedule"`
WakeupInterval time.Duration `yaml:"WakeupInterval" json:"WakeupInterval" flag:"wakeup-interval"`
@@ -213,6 +214,10 @@ func NewOptions(ctx *cli.Context) *Options {
c.Copyright = ctx.App.Copyright
c.Version = ctx.App.Version
// Set defaults.
c.SidecarYaml = true
c.BackupAlbums = true
// Load defaults from YAML file?
if defaultsYaml := ctx.GlobalString("defaults-yaml"); defaultsYaml == "" {
log.Tracef("config: defaults yaml file not specified")

View File

@@ -63,7 +63,7 @@ func (c *Config) Report() (rows [][]string, cols []string) {
{"storage-path", c.StoragePath()},
{"users-storage-path", c.UsersStoragePath()},
{"sidecar-path", c.SidecarPath()},
{"albums-path", c.AlbumsPath()},
{"sidecar-yaml", fmt.Sprintf("%t", c.SidecarYaml())},
{"cache-path", c.CachePath()},
{"cmd-cache-path", c.CmdCachePath()},
{"media-cache-path", c.MediaCachePath()},
@@ -78,11 +78,13 @@ func (c *Config) Report() (rows [][]string, cols []string) {
{"temp-path", c.TempPath()},
// Backups.
{"backup-path", c.BackupPath()},
{"backup-index", fmt.Sprintf("%t", c.BackupIndex())},
{"backup-albums", fmt.Sprintf("%t", c.BackupAlbums())},
{"backup-retain", fmt.Sprintf("%d", c.BackupRetain())},
{"backup-path", c.BackupBasePath()},
{"backup-schedule", c.BackupSchedule()},
{"backup-retain", fmt.Sprintf("%d", c.BackupRetain())},
{"backup-index", fmt.Sprintf("%t", c.BackupIndex())},
{"backup-index-path", c.BackupIndexPath()},
{"backup-albums", fmt.Sprintf("%t", c.BackupAlbums())},
{"backup-albums-path", c.BackupAlbumsPath()},
// IndexWorkers.
{"index-workers", fmt.Sprintf("%d", c.IndexWorkers())},
@@ -97,7 +99,6 @@ func (c *Config) Report() (rows [][]string, cols []string) {
{"disable-webdav", fmt.Sprintf("%t", c.DisableWebDAV())},
{"disable-settings", fmt.Sprintf("%t", c.DisableSettings())},
{"disable-places", fmt.Sprintf("%t", c.DisablePlaces())},
{"disable-backups", fmt.Sprintf("%t", c.DisableBackups())},
{"disable-tensorflow", fmt.Sprintf("%t", c.DisableTensorFlow())},
{"disable-faces", fmt.Sprintf("%t", c.DisableFaces())},
{"disable-classification", fmt.Sprintf("%t", c.DisableClassification())},

View File

@@ -198,6 +198,30 @@ func NewTestErrorConfig() *Config {
return c
}
// NewTestContext creates a new CLI test context with the flags and arguments provided.
func NewTestContext(args []string) *cli.Context {
// Create new command-line app.
app := cli.NewApp()
app.Usage = "PhotoPrism®"
app.Version = "test"
app.Copyright = "(c) 2018-2024 PhotoPrism UG. All rights reserved."
app.EnableBashCompletion = true
app.Flags = Flags.Cli()
app.Metadata = map[string]interface{}{
"Name": "PhotoPrism",
"About": "PhotoPrism®",
"Edition": "ce",
"Version": "test",
}
// Parse command arguments.
flags := flag.NewFlagSet("test", 0)
LogErr(flags.Parse(args))
// Create and return new context.
return cli.NewContext(app, flags, nil)
}
// CliTestContext returns a CLI context for testing.
func CliTestContext() *cli.Context {
config := NewTestOptions("config-cli")
@@ -207,6 +231,7 @@ func CliTestContext() *cli.Context {
globalSet.String("admin-password", config.DarktableBin, "doc")
globalSet.String("storage-path", config.StoragePath, "doc")
globalSet.String("sidecar-path", config.SidecarPath, "doc")
globalSet.Bool("sidecar-yaml", config.SidecarYaml, "doc")
globalSet.String("assets-path", config.AssetsPath, "doc")
globalSet.String("originals-path", config.OriginalsPath, "doc")
globalSet.String("import-path", config.OriginalsPath, "doc")
@@ -235,6 +260,7 @@ func CliTestContext() *cli.Context {
LogErr(c.Set("admin-password", config.AdminPassword))
LogErr(c.Set("storage-path", config.StoragePath))
LogErr(c.Set("sidecar-path", config.SidecarPath))
LogErr(c.Set("sidecar-yaml", fmt.Sprintf("%t", config.SidecarYaml)))
LogErr(c.Set("assets-path", config.AssetsPath))
LogErr(c.Set("originals-path", config.OriginalsPath))
LogErr(c.Set("import-path", config.ImportPath))

View File

@@ -2,22 +2,22 @@ package photoprism
import (
"path/filepath"
"regexp"
"sync"
"time"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/pkg/clean"
"github.com/photoprism/photoprism/pkg/fs"
)
var backupAlbumsLatest = time.Time{}
var backupAlbumsMutex = sync.Mutex{}
// BackupAlbums creates a YAML file backup of all albums.
func BackupAlbums(backupPath string, force bool) (count int, err error) {
c := Config()
if !c.BackupYaml() && !force {
log.Debugf("backup: album yaml files disabled")
return count, nil
}
// Make sure only one backup/restore operation is running at a time.
backupAlbumsMutex.Lock()
defer backupAlbumsMutex.Unlock()
albums, queryErr := query.Albums(0, 1000000)
@@ -26,10 +26,26 @@ func BackupAlbums(backupPath string, force bool) (count int, err error) {
}
if !fs.PathExists(backupPath) {
backupPath = c.AlbumsPath()
backupPath = Config().BackupAlbumsPath()
}
log.Tracef("creating album YAML files in %s", clean.Log(filepath.Base(backupPath)))
var latest time.Time
if !force {
latest = backupAlbumsLatest
}
for _, a := range albums {
if !force && a.UpdatedAt.Before(backupAlbumsLatest) {
continue
}
if a.UpdatedAt.After(latest) {
latest = a.UpdatedAt
}
fileName := a.YamlFileName(backupPath)
if saveErr := a.SaveAsYaml(fileName); saveErr != nil {
@@ -41,64 +57,7 @@ func BackupAlbums(backupPath string, force bool) (count int, err error) {
}
}
backupAlbumsLatest = latest
return count, err
}
// RestoreAlbums restores all album YAML file backups.
func RestoreAlbums(backupPath string, force bool) (count int, result error) {
c := Config()
if !c.BackupYaml() && !force {
log.Debugf("restore: album yaml files disabled")
return count, nil
}
existing, err := query.Albums(0, 1)
if err != nil {
return count, err
}
if len(existing) > 0 && !force {
log.Debugf("restore: album yaml files disabled")
return count, nil
}
if !fs.PathExists(backupPath) {
backupPath = c.AlbumsPath()
}
albums, err := filepath.Glob(regexp.QuoteMeta(backupPath) + "/**/*.yml")
if oAlbums, oErr := filepath.Glob(regexp.QuoteMeta(c.OriginalsAlbumsPath()) + "/**/*.yml"); oErr == nil {
err = nil
albums = append(albums, oAlbums...)
}
if err != nil {
return count, err
}
if len(albums) == 0 {
return count, nil
}
for _, fileName := range albums {
a := entity.Album{}
if err = a.LoadFromYaml(fileName); err != nil {
log.Errorf("restore: %s in %s", err, clean.Log(filepath.Base(fileName)))
result = err
} else if a.AlbumType == "" || len(a.Photos) == 0 && a.AlbumFilter == "" {
log.Debugf("restore: skipping %s", clean.Log(filepath.Base(fileName)))
} else if found := a.Find(); found != nil {
log.Infof("%s: %s already exists", found.AlbumType, clean.Log(found.AlbumTitle))
} else if err = a.Create(); err != nil {
log.Errorf("%s: %s in %s", a.AlbumType, err, clean.Log(filepath.Base(fileName)))
} else {
count++
}
}
return count, result
}

View File

@@ -10,6 +10,7 @@ import (
"regexp"
"sort"
"strings"
"sync"
"time"
"github.com/dustin/go-humanize/english"
@@ -19,13 +20,19 @@ import (
"github.com/photoprism/photoprism/pkg/fs"
)
var backupIndexMutex = sync.Mutex{}
// BackupIndex creates an SQL backup dump with the specified file and path name.
func BackupIndex(backupPath, fileName string, toStdOut, force bool, retain int) (err error) {
// Make sure only one backup/restore operation is running at a time.
backupIndexMutex.Lock()
defer backupIndexMutex.Unlock()
c := Config()
if !toStdOut {
if backupPath == "" {
backupPath = filepath.Join(c.BackupPath(), c.DatabaseDriver())
backupPath = c.BackupIndexPath()
}
// Create the backup path if it does not already exist.

View File

@@ -992,10 +992,12 @@ func (ind *Index) UserMediaFile(m *MediaFile, o IndexOptions, originalName, phot
return result
}
if file.FilePrimary && Config().BackupYaml() {
// Write YAML sidecar file (optional).
// Create backup of picture metadata in sidecar YAML file.
if file.FilePrimary && Config().SidecarYaml() {
// Get YAML file name.
yamlFile := photo.YamlFileName(Config().OriginalsPath(), Config().SidecarPath())
// Save backup to file.
if err = photo.SaveAsYaml(yamlFile); err != nil {
log.Errorf("index: %s in %s (update yaml)", err.Error(), logName)
} else {

View File

@@ -0,0 +1,74 @@
package photoprism
import (
"path/filepath"
"regexp"
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/query"
"github.com/photoprism/photoprism/pkg/clean"
"github.com/photoprism/photoprism/pkg/fs"
)
// RestoreAlbums restores all album YAML file backups.
func RestoreAlbums(backupPath string, force bool) (count int, result error) {
// Make sure only one backup/restore operation is running at a time.
backupAlbumsMutex.Lock()
defer backupAlbumsMutex.Unlock()
c := Config()
if !c.BackupAlbums() && !force {
log.Debugf("restore: album yaml file backups are disabled")
return count, nil
}
existing, err := query.Albums(0, 1)
if err != nil {
return count, err
}
if len(existing) > 0 && !force {
log.Debugf("restore: found existing albums, use the force option to overwrite")
return count, nil
}
if !fs.PathExists(backupPath) {
backupPath = c.BackupAlbumsPath()
}
albums, err := filepath.Glob(regexp.QuoteMeta(backupPath) + "/**/*.yml")
if oAlbums, oErr := filepath.Glob(regexp.QuoteMeta(c.OriginalsAlbumsPath()) + "/**/*.yml"); oErr == nil {
err = nil
albums = append(albums, oAlbums...)
}
if err != nil {
return count, err
}
if len(albums) == 0 {
return count, nil
}
for _, fileName := range albums {
a := entity.Album{}
if err = a.LoadFromYaml(fileName); err != nil {
log.Errorf("restore: %s in %s", err, clean.Log(filepath.Base(fileName)))
result = err
} else if a.AlbumType == "" || len(a.Photos) == 0 && a.AlbumFilter == "" {
log.Debugf("restore: skipping %s", clean.Log(filepath.Base(fileName)))
} else if found := a.Find(); found != nil {
log.Infof("%s: %s already exists", found.AlbumType, clean.Log(found.AlbumTitle))
} else if err = a.Create(); err != nil {
log.Errorf("%s: %s in %s", a.AlbumType, err, clean.Log(filepath.Base(fileName)))
} else {
count++
}
}
return count, result
}

View File

@@ -22,12 +22,16 @@ const SqlBackupFileNamePattern = "[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9].sql
// RestoreIndex restores the index from an SQL backup dump with the specified file and path name.
func RestoreIndex(backupPath, fileName string, fromStdIn, force bool) (err error) {
// Make sure only one backup/restore operation is running at a time.
backupIndexMutex.Lock()
defer backupIndexMutex.Unlock()
c := Config()
// If empty, use default backup file name.
if !fromStdIn && fileName == "" {
if backupPath == "" {
backupPath = filepath.Join(c.BackupPath(), c.DatabaseDriver())
backupPath = c.BackupIndexPath()
}
files, globErr := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), SqlBackupFileNamePattern))
@@ -58,7 +62,7 @@ func RestoreIndex(backupPath, fileName string, fromStdIn, force bool) (err error
if counts.Photos == 0 {
// Do nothing;
} else if !force {
return fmt.Errorf("existing index found with %d pictures, use the force option to replace it", counts.Photos)
return fmt.Errorf("found existing index with %d pictures, use the force option to replace it", counts.Photos)
} else {
log.Warnf("replacing the existing index with %d pictures", counts.Photos)
}

View File

@@ -3,7 +3,6 @@ package workers
import (
"errors"
"fmt"
"path/filepath"
"runtime/debug"
"time"
@@ -11,6 +10,7 @@ import (
"github.com/photoprism/photoprism/internal/entity"
"github.com/photoprism/photoprism/internal/mutex"
"github.com/photoprism/photoprism/internal/photoprism"
"github.com/photoprism/photoprism/pkg/clean"
)
// Backup represents a background backup worker.
@@ -54,7 +54,7 @@ func (w *Backup) Start(index, albums bool, force bool, retain int) (err error) {
// Start creating backups.
start := time.Now()
backupPath := filepath.Join(w.conf.BackupPath(), w.conf.DatabaseDriver())
backupPath := w.conf.BackupIndexPath()
if index && albums {
log.Infof("backup: creating index and album backups")
@@ -76,13 +76,16 @@ func (w *Backup) Start(index, albums bool, force bool, retain int) (err error) {
}
// Create album YAML file backup.
if !albums {
// Skip.
} else if count, backupErr := photoprism.BackupAlbums(w.conf.AlbumsPath(), force); backupErr != nil {
if albums {
albumsBackupPath := w.conf.BackupAlbumsPath()
log.Infof("creating album YAML files in %s", clean.Log(albumsBackupPath))
if count, backupErr := photoprism.BackupAlbums(albumsBackupPath, force); backupErr != nil {
log.Errorf("backup: %s (albums)", backupErr.Error())
} else if count > 0 {
log.Debugf("backup: %d albums saved as yaml files", count)
}
}
// Update time when worker was last executed.
w.lastRun = entity.TimeStamp()

View File

@@ -6,30 +6,54 @@ import (
// Type omits invalid runes, ensures a maximum length of 32 characters, and returns the result.
func Type(s string) string {
if s == "" {
return s
}
return Clip(ASCII(s), ClipType)
}
// TypeLower converts a type string to lowercase, omits invalid runes, and shortens it if needed.
func TypeLower(s string) string {
if s == "" {
return s
}
return Type(strings.ToLower(s))
}
// TypeLowerUnderscore converts a string to a lowercase type string and replaces spaces with underscores.
func TypeLowerUnderscore(s string) string {
if s == "" {
return s
}
return strings.ReplaceAll(TypeLower(s), " ", "_")
}
// ShortType omits invalid runes, ensures a maximum length of 8 characters, and returns the result.
func ShortType(s string) string {
if s == "" {
return s
}
return Clip(ASCII(s), ClipShortType)
}
// ShortTypeLower converts a short type string to lowercase, omits invalid runes, and shortens it if needed.
func ShortTypeLower(s string) string {
if s == "" {
return s
}
return ShortType(strings.ToLower(s))
}
// ShortTypeLowerUnderscore converts a string to a short lowercase type string and replaces spaces with underscores.
func ShortTypeLowerUnderscore(s string) string {
if s == "" {
return s
}
return strings.ReplaceAll(ShortTypeLower(s), " ", "_")
}

View File

@@ -88,12 +88,15 @@ func Writable(path string) bool {
if path == "" {
return false
}
return syscall.Access(path, syscall.O_RDWR) == nil
}
// PathWritable tests if a path exists and is writable.
func PathWritable(path string) bool {
if !PathExists(path) {
if path == "" {
return false
} else if !PathExists(path) {
return false
}