Backups: Add config option to limit the number of backups to keep #4243

PHOTOPRISM_BACKUP_RETAIN lets to specify the number of index database
dumps to keep (backup filenames are in the format "YYYY-MM-DD.sql").

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/commands/backup.go

@@ -9,6 +9,7 @@ import (
 	"github.com/dustin/go-humanize/english"
 	"github.com/urfave/cli"
 
+	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/photoprism"
 	"github.com/photoprism/photoprism/pkg/clean"
 	"github.com/photoprism/photoprism/pkg/fs"
@@ -23,7 +24,7 @@ const backupDescription = "A user-defined filename or - for stdout can be passed
 var BackupCommand = cli.Command{
 	Name:        "backup",
 	Description: backupDescription,
-	Usage:       "Creates an index backup and optionally album YAML files organized by type",
+	Usage:       "Creates an index database dump and/or album YAML file backups",
 	ArgsUsage:   "[filename]",
 	Flags:       backupFlags,
 	Action:      backupAction,
@@ -32,23 +33,28 @@ var BackupCommand = cli.Command{
 var backupFlags = []cli.Flag{
 	cli.BoolFlag{
 		Name:  "force, f",
-		Usage: "replace existing files",
+		Usage: "replace existing index backup files",
 	},
 	cli.BoolFlag{
 		Name:  "albums, a",
-		Usage: "create album YAML files organized by type",
+		Usage: "create album YAML file backups in the configured backup path",
 	},
 	cli.StringFlag{
 		Name:  "albums-path",
-		Usage: "custom album files `PATH`",
+		Usage: "custom `PATH` for creating album backups",
 	},
 	cli.BoolFlag{
 		Name:  "index, i",
-		Usage: "create index backup",
+		Usage: "create index backup in the configured backup path (stdout if - is passed as first argument)",
 	},
 	cli.StringFlag{
 		Name:  "index-path",
-		Usage: "custom index backup `PATH`",
+		Usage: "custom `PATH` for creating index backups",
+	},
+	cli.IntFlag{
+		Name:  "retain, r",
+		Usage: "`NUMBER` of index backups to keep (-1 to keep all)",
+		Value: config.DefaultBackupRetain,
 	},
 }
 
@@ -61,6 +67,7 @@ func backupAction(ctx *cli.Context) error {
 	albumsPath := ctx.String("albums-path")
 	backupAlbums := ctx.Bool("albums") || albumsPath != ""
 	force := ctx.Bool("force")
+	retain := ctx.Int("retain")
 
 	if !backupIndex && !backupAlbums {
 		return cli.ShowSubcommandHelp(ctx)
@@ -95,8 +102,8 @@ func backupAction(ctx *cli.Context) error {
 			fileName = filepath.Join(backupPath, backupFile)
 		}
 
-		if err = photoprism.BackupIndex(backupPath, fileName, fileName == "-", force); err != nil {
-			return fmt.Errorf("failed to create %s: %w", clean.Log(fileName), err)
+		if err = photoprism.BackupIndex(backupPath, fileName, fileName == "-", force, retain); err != nil {
+			return fmt.Errorf("failed to create index backup: %w", err)
 		}
 	}
 

internal/commands/restore.go

@@ -1,21 +1,12 @@
 package commands
 
 import (
-	"bytes"
 	"context"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"regexp"
 	"time"
 
 	"github.com/dustin/go-humanize/english"
 	"github.com/urfave/cli"
 
-	"github.com/photoprism/photoprism/internal/config"
-	"github.com/photoprism/photoprism/internal/entity"
 	"github.com/photoprism/photoprism/internal/get"
 	"github.com/photoprism/photoprism/internal/photoprism"
 	"github.com/photoprism/photoprism/pkg/clean"
@@ -30,7 +21,7 @@ const restoreDescription = "A user-defined filename or - for stdin can be passed
 var RestoreCommand = cli.Command{
 	Name:        "restore",
 	Description: restoreDescription,
-	Usage:       "Restores the index from a backup and optionally albums from YAML files",
+	Usage:       "Restores the index from a database dump and/or album YAML file backups",
 	ArgsUsage:   "[filename]",
 	Flags:       restoreFlags,
 	Action:      restoreAction,
@@ -39,23 +30,23 @@ var RestoreCommand = cli.Command{
 var restoreFlags = []cli.Flag{
 	cli.BoolFlag{
 		Name:  "force, f",
-		Usage: "replace existing index",
+		Usage: "replace existing index schema and data",
 	},
 	cli.BoolFlag{
 		Name:  "albums, a",
-		Usage: "restore albums from YAML files",
+		Usage: "restore album YAML file backups from the configured backup path",
 	},
 	cli.StringFlag{
 		Name:  "albums-path",
-		Usage: "custom album files `PATH`",
+		Usage: "custom `PATH` for restoring album backups",
 	},
 	cli.BoolFlag{
 		Name:  "index, i",
-		Usage: "restore index from backup",
+		Usage: "restore index from the latest backup in the configured backup path (or the file passed as first argument)",
 	},
 	cli.StringFlag{
 		Name:  "index-path",
-		Usage: "custom index backup `PATH`",
+		Usage: "custom `PATH` for restoring index backups",
 	},
 }
 
@@ -65,7 +56,7 @@ func restoreAction(ctx *cli.Context) error {
 	indexFileName := ctx.Args().First()
 	indexPath := ctx.String("index-path")
 	restoreIndex := ctx.Bool("index") || indexFileName != "" || indexPath != ""
-
+	force := ctx.Bool("force")
 	albumsPath := ctx.String("albums-path")
 	restoreAlbums := ctx.Bool("albums") || albumsPath != ""
 
@@ -87,109 +78,13 @@ func restoreAction(ctx *cli.Context) error {
 	conf.RegisterDb()
 	defer conf.Shutdown()
 
-	if restoreIndex {
-		// If empty, use default backup file name.
-		if indexFileName == "" {
-			if indexPath == "" {
-				indexPath = filepath.Join(conf.BackupPath(), conf.DatabaseDriver())
-			}
-
-			matches, err := filepath.Glob(filepath.Join(regexp.QuoteMeta(indexPath), "*.sql"))
-
-			if err != nil {
+	// Restore index from specified file?
+	if !restoreIndex {
+		// Do nothing.
+	} else if err = photoprism.RestoreIndex(indexPath, indexFileName, indexFileName == "-", force); err != nil {
 		return err
 	}
 
-			if len(matches) == 0 {
-				log.Errorf("no backup files found in %s", indexPath)
-				return nil
-			}
-
-			indexFileName = matches[len(matches)-1]
-		}
-
-		counts := struct{ Photos int }{}
-
-		conf.Db().Unscoped().Table("photos").
-			Select("COUNT(*) AS photos").
-			Take(&counts)
-
-		if counts.Photos == 0 {
-			// Do nothing;
-		} else if !ctx.Bool("force") {
-			return fmt.Errorf("found exisisting index with %d pictures, use --force to replace it", counts.Photos)
-		} else {
-			log.Warnf("replacing existing index with %d pictures", counts.Photos)
-		}
-
-		tables := entity.Entities
-
-		var cmd *exec.Cmd
-
-		switch conf.DatabaseDriver() {
-		case config.MySQL, config.MariaDB:
-			cmd = exec.Command(
-				conf.MariadbBin(),
-				"--protocol", "tcp",
-				"-h", conf.DatabaseHost(),
-				"-P", conf.DatabasePortString(),
-				"-u", conf.DatabaseUser(),
-				"-p"+conf.DatabasePassword(),
-				"-f",
-				conf.DatabaseName(),
-			)
-		case config.SQLite3:
-			log.Infoln("dropping existing tables")
-			tables.Drop(conf.Db())
-			cmd = exec.Command(
-				conf.SqliteBin(),
-				conf.DatabaseFile(),
-			)
-		default:
-			return fmt.Errorf("unsupported database type: %s", conf.DatabaseDriver())
-		}
-
-		// Read from stdin or file.
-		var f *os.File
-		if indexFileName == "-" {
-			log.Infof("restoring index from stdin")
-			f = os.Stdin
-		} else if f, err = os.OpenFile(indexFileName, os.O_RDONLY, 0); err != nil {
-			return fmt.Errorf("failed to open %s: %s", clean.Log(indexFileName), err)
-		} else {
-			log.Infof("restoring index from %s", clean.Log(indexFileName))
-			defer f.Close()
-		}
-
-		var stderr bytes.Buffer
-		var stdin io.WriteCloser
-		cmd.Stderr = &stderr
-		cmd.Stdout = os.Stdout
-		stdin, err = cmd.StdinPipe()
-
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		go func() {
-			defer stdin.Close()
-			if _, err = io.Copy(stdin, f); err != nil {
-				log.Errorf(err.Error())
-			}
-		}()
-
-		// Log exact command for debugging in trace mode.
-		log.Trace(cmd.String())
-
-		// Run backup command.
-		if err := cmd.Run(); err != nil {
-			if stderr.String() != "" {
-				log.Debugln(stderr.String())
-				log.Warnf("index could not be restored completely")
-			}
-		}
-	}
-
 	log.Infoln("migrating index database schema")
 
 	conf.InitDb()

internal/commands/show_config_options.go

@@ -49,6 +49,7 @@ func showConfigOptionsAction(ctx *cli.Context) error {
 		{Start: "PHOTOPRISM_ADMIN_PASSWORD", Title: "Authentication"},
 		{Start: "PHOTOPRISM_LOG_LEVEL", Title: "Logging"},
 		{Start: "PHOTOPRISM_CONFIG_PATH", Title: "Storage"},
+		{Start: "PHOTOPRISM_BACKUP_PATH", Title: "Backups"},
 		{Start: "PHOTOPRISM_INDEX_WORKERS, PHOTOPRISM_WORKERS", Title: "Index Workers"},
 		{Start: "PHOTOPRISM_READONLY", Title: "Feature Flags"},
 		{Start: "PHOTOPRISM_DEFAULT_LOCALE", Title: "Customization"},

internal/config/config_backup.go

@@ -22,7 +22,7 @@ func (c *Config) BackupPath() string {
 	return filepath.Join(c.StoragePath(), "backup")
 }
 
-// BackupIndex checks if index SQL database dumps should be created based on the configured schedule.
+// BackupIndex checks if SQL database dumps should be created based on the configured schedule.
 func (c *Config) BackupIndex() bool {
 	return c.options.BackupIndex
 }

internal/config/config_db.go

@@ -102,7 +102,7 @@ func (c *Config) DatabaseDsn() string {
 
 // DatabaseFile returns the filename part of a sqlite database DSN.
 func (c *Config) DatabaseFile() string {
-	fileName, _, _ := strings.Cut(c.DatabaseDsn(), "?")
+	fileName, _, _ := strings.Cut(strings.TrimPrefix(c.DatabaseDsn(), "file:"), "?")
 	return fileName
 }
 

internal/config/flags.go

@@ -183,12 +183,12 @@ var Flags = CliFlags{
 		}}, {
 		Flag: cli.StringFlag{
 			Name:   "backup-path, ba",
-			Usage:  "custom backup `PATH` for index backup files *optional*",
+			Usage:  "custom default `PATH` for creating and restoring index backups *optional*",
 			EnvVar: EnvVar("BACKUP_PATH"),
 		}}, {
 		Flag: cli.BoolFlag{
 			Name:   "backup-index",
-			Usage:  "create index SQL database dumps based on the configured schedule",
+			Usage:  "create index backups based on the configured schedule",
 			EnvVar: EnvVar("BACKUP_INDEX"),
 		}}, {
 		Flag: cli.BoolFlag{
@@ -198,7 +198,7 @@ var Flags = CliFlags{
 		}}, {
 		Flag: cli.IntFlag{
 			Name:   "backup-retain",
-			Usage:  "maximum `NUMBER` of SQL database dumps to keep (-1 to keep all)",
+			Usage:  "`NUMBER` of index backups to keep (-1 to keep all)",
 			Value:  DefaultBackupRetain,
 			EnvVar: EnvVar("BACKUP_RETAIN"),
 		}}, {

internal/photoprism/backup_index.go

@@ -7,19 +7,23 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
+	"sort"
 	"strings"
 	"time"
 
+	"github.com/dustin/go-humanize/english"
+
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/pkg/clean"
 	"github.com/photoprism/photoprism/pkg/fs"
 )
 
 // BackupIndex creates an SQL backup dump with the specified file and path name.
-func BackupIndex(backupPath, fileName string, stdOut, force bool) (err error) {
+func BackupIndex(backupPath, fileName string, toStdOut, force bool, retain int) (err error) {
 	c := Config()
 
-	if !stdOut {
+	if !toStdOut {
 		if backupPath == "" {
 			backupPath = filepath.Join(c.BackupPath(), c.DatabaseDriver())
 		}
@@ -31,7 +35,7 @@ func BackupIndex(backupPath, fileName string, stdOut, force bool) (err error) {
 
 		// Check if the backup path is writable.
 		if !fs.PathWritable(backupPath) {
-			return fmt.Errorf("backup: path is not writable")
+			return fmt.Errorf("backup path is not writable")
 		}
 
 		if fileName == "" {
@@ -40,9 +44,9 @@ func BackupIndex(backupPath, fileName string, stdOut, force bool) (err error) {
 		}
 
 		if _, err = os.Stat(fileName); err == nil && !force {
-			return fmt.Errorf("%s already exists", clean.Log(fileName))
+			return fmt.Errorf("%s already exists", clean.Log(filepath.Base(fileName)))
 		} else if err == nil {
-			log.Warnf("replacing existing backup")
+			log.Warnf("replacing existing index backup")
 		}
 
 		// Create backup path if not exists.
@@ -67,6 +71,10 @@ func BackupIndex(backupPath, fileName string, stdOut, force bool) (err error) {
 			c.DatabaseName(),
 		)
 	case config.SQLite3:
+		if !fs.FileExistsNotEmpty(c.DatabaseFile()) {
+			return fmt.Errorf("sqlite database %s not found", clean.LogQuote(c.DatabaseFile()))
+		}
+
 		cmd = exec.Command(
 			c.SqliteBin(),
 			c.DatabaseFile(),
@@ -78,13 +86,13 @@ func BackupIndex(backupPath, fileName string, stdOut, force bool) (err error) {
 
 	// Write to stdout or file.
 	var f *os.File
-	if fileName == "-" {
-		log.Infof("writing backup to stdout")
+	if toStdOut {
+		log.Infof("writing index backup to stdout")
 		f = os.Stdout
 	} else if f, err = os.OpenFile(fileName, os.O_TRUNC|os.O_RDWR|os.O_CREATE, fs.ModeFile); err != nil {
 		return fmt.Errorf("failed to create %s: %s", clean.Log(fileName), err)
 	} else {
-		log.Infof("writing backup to %s", clean.Log(fileName))
+		log.Infof("creating index backup in %s", clean.Log(filepath.Base(fileName)))
 		defer f.Close()
 	}
 
@@ -104,5 +112,32 @@ func BackupIndex(backupPath, fileName string, stdOut, force bool) (err error) {
 		return cmdErr
 	}
 
+	// Delete old backups if the number of backup files to keep has been specified.
+	if !toStdOut && backupPath != "" && retain > 0 {
+		files, globErr := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), SqlBackupFileNamePattern))
+
+		if globErr != nil {
+			return globErr
+		}
+
+		if len(files) == 0 {
+			return fmt.Errorf("found no index backups files in %s", backupPath)
+		} else if len(files) <= retain {
+			return nil
+		}
+
+		sort.Strings(files)
+
+		log.Infof("retaining %s", english.Plural(retain, "index backup", "index backups"))
+
+		for i := 0; i < len(files)-retain; i++ {
+			if err = os.Remove(files[i]); err != nil {
+				return err
+			} else {
+				log.Infof("removed old backup file %s", clean.Log(filepath.Base(files[i])))
+			}
+		}
+	}
+
 	return nil
 }

internal/photoprism/restore_index.go

@@ -0,0 +1,137 @@
+package photoprism
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/pkg/clean"
+	"github.com/photoprism/photoprism/pkg/fs"
+)
+
+const SqlBackupFileNamePattern = "[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9].sql"
+
+// RestoreIndex restores the index from an SQL backup dump with the specified file and path name.
+func RestoreIndex(backupPath, fileName string, fromStdIn, force bool) (err error) {
+	c := Config()
+
+	// If empty, use default backup file name.
+	if !fromStdIn && fileName == "" {
+		if backupPath == "" {
+			backupPath = filepath.Join(c.BackupPath(), c.DatabaseDriver())
+		}
+
+		files, globErr := filepath.Glob(filepath.Join(regexp.QuoteMeta(backupPath), SqlBackupFileNamePattern))
+
+		if globErr != nil {
+			return globErr
+		}
+
+		if len(files) == 0 {
+			return fmt.Errorf("found no backups files in %s", backupPath)
+		}
+
+		sort.Strings(files)
+
+		fileName = files[len(files)-1]
+
+		if !fs.FileExistsNotEmpty(fileName) {
+			return fmt.Errorf("no backup found in %s", filepath.Base(fileName))
+		}
+	}
+
+	counts := struct{ Photos int }{}
+
+	c.Db().Unscoped().Table("photos").
+		Select("COUNT(*) AS photos").
+		Take(&counts)
+
+	if counts.Photos == 0 {
+		// Do nothing;
+	} else if !force {
+		return fmt.Errorf("existing index found with %d pictures, use the force option to replace it", counts.Photos)
+	} else {
+		log.Warnf("replacing the existing index with %d pictures", counts.Photos)
+	}
+
+	tables := entity.Entities
+
+	var cmd *exec.Cmd
+
+	switch c.DatabaseDriver() {
+	case config.MySQL, config.MariaDB:
+		cmd = exec.Command(
+			c.MariadbBin(),
+			"--protocol", "tcp",
+			"-h", c.DatabaseHost(),
+			"-P", c.DatabasePortString(),
+			"-u", c.DatabaseUser(),
+			"-p"+c.DatabasePassword(),
+			"-f",
+			c.DatabaseName(),
+		)
+	case config.SQLite3:
+		log.Infoln("dropping existing tables")
+		tables.Drop(c.Db())
+		cmd = exec.Command(
+			c.SqliteBin(),
+			c.DatabaseFile(),
+		)
+	default:
+		return fmt.Errorf("unsupported database type: %s", c.DatabaseDriver())
+	}
+
+	// Read from stdin or file.
+	var f *os.File
+	if fromStdIn {
+		log.Infof("restoring index from stdin")
+		f = os.Stdin
+	} else if f, err = os.OpenFile(fileName, os.O_RDONLY, 0); err != nil {
+		return fmt.Errorf("failed to open %s: %s", clean.Log(fileName), err)
+	} else {
+		log.Infof("restoring index from %s", clean.Log(fileName))
+		defer f.Close()
+	}
+
+	var stderr bytes.Buffer
+	var stdin io.WriteCloser
+	cmd.Stderr = &stderr
+	cmd.Stdout = os.Stdout
+	stdin, err = cmd.StdinPipe()
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	go func() {
+		defer stdin.Close()
+		if _, err = io.Copy(stdin, f); err != nil {
+			log.Errorf(err.Error())
+		}
+	}()
+
+	// Log exact command for debugging in trace mode.
+	log.Trace(cmd.String())
+
+	// Run restore command.
+	if cmdErr := cmd.Run(); cmdErr != nil {
+		log.Errorf("failed to restore index")
+
+		if errStr := strings.TrimSpace(stderr.String()); errStr != "" {
+			return errors.New(errStr)
+		}
+
+		return cmdErr
+	}
+
+	return nil
+}

internal/workers/backup.go

@@ -26,13 +26,13 @@ func NewBackup(conf *config.Config) *Backup {
 
 // StartScheduled starts a scheduled run of the backup worker based on the current configuration.
 func (w *Backup) StartScheduled() {
-	if err := w.Start(w.conf.BackupIndex(), w.conf.BackupAlbums(), true); err != nil {
+	if err := w.Start(w.conf.BackupIndex(), w.conf.BackupAlbums(), true, w.conf.BackupRetain()); err != nil {
 		log.Errorf("scheduler: %s (backup)", err)
 	}
 }
 
 // Start creates index and album backups based on the current configuration.
-func (w *Backup) Start(index, albums, force bool) (err error) {
+func (w *Backup) Start(index, albums bool, force bool, retain int) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
 			err = fmt.Errorf("backup: %s (worker panic)\nstack: %s", r, debug.Stack())
@@ -67,8 +67,8 @@ func (w *Backup) Start(index, albums, force bool) (err error) {
 	// Create index database backup.
 	if !index {
 		// Skip.
-	} else if err = photoprism.BackupIndex(backupPath, "", false, force); err != nil {
-		log.Errorf("backup: %s (backup index)", err)
+	} else if err = photoprism.BackupIndex(backupPath, "", false, force, retain); err != nil {
+		log.Errorf("backup: %s (index)", err)
 	}
 
 	if mutex.BackupWorker.Canceled() {
@@ -79,7 +79,7 @@ func (w *Backup) Start(index, albums, force bool) (err error) {
 	if !albums {
 		// Skip.
 	} else if count, backupErr := photoprism.BackupAlbums(w.conf.AlbumsPath(), force); backupErr != nil {
-		log.Errorf("backup: %s (backup albums)", backupErr.Error())
+		log.Errorf("backup: %s (albums)", backupErr.Error())
 	} else if count > 0 {
 		log.Debugf("backup: %d albums saved as yaml files", count)
 	}

internal/workers/backup_test.go

@@ -23,19 +23,19 @@ func TestBackup_Start(t *testing.T) {
 	}
 
 	// Mutex should prevent worker from starting.
-	if err := worker.Start(true, true, true); err == nil {
+	if err := worker.Start(true, true, true, 2); err == nil {
 		t.Fatal("error expected")
 	}
 
 	mutex.BackupWorker.Stop()
 
 	// Start worker.
-	if err := worker.Start(true, true, false); err != nil {
+	if err := worker.Start(true, true, true, 2); err != nil {
 		t.Fatal(err)
 	}
 
 	// Rerun worker.
-	if err := worker.Start(true, true, false); err != nil {
+	if err := worker.Start(true, true, false, 2); err != nil {
 		t.Fatal(err)
 	}
 }