//go:build !plan9 // Package create implements 'rclone archive create'. package create import ( "context" "errors" "fmt" "io" "os" "path" "sort" "strings" "time" "github.com/mholt/archives" "github.com/rclone/rclone/cmd" "github.com/rclone/rclone/cmd/archive" "github.com/rclone/rclone/cmd/archive/files" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/flags" "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/walk" "github.com/spf13/cobra" ) var ( fullPath = false prefix = "" format = "" ) func init() { flagSet := Command.Flags() flags.BoolVarP(flagSet, &fullPath, "full-path", "", fullPath, "Set prefix for files in archive to source path", "") flags.StringVarP(flagSet, &prefix, "prefix", "", prefix, "Set prefix for files in archive to entered value or source path", "") flags.StringVarP(flagSet, &format, "format", "", format, "Create the archive with format or guess from extension.", "") archive.Command.AddCommand(Command) } // Command - create var Command = &cobra.Command{ Use: "create [flags] []", Short: `Archive source file(s) to destination.`, // Warning! "!" will be replaced by backticks below Long: strings.ReplaceAll(` Creates an archive from the files in source:path and saves the archive to dest:path. If dest:path is missing, it will write to the console. The valid formats for the !--format! flag are listed below. If !--format! is not set rclone will guess it from the extension of dest:path. | Format | Extensions | |:-------|:-----------| | zip | .zip | | tar | .tar | | tar.gz | .tar.gz, .tgz, .taz | | tar.bz2| .tar.bz2, .tb2, .tbz, .tbz2, .tz2 | | tar.lz | .tar.lz | | tar.lz4| .tar.lz4 | | tar.xz | .tar.xz, .txz | | tar.zst| .tar.zst, .tzst | | tar.br | .tar.br | | tar.sz | .tar.sz | | tar.mz | .tar.mz | The !--prefix! and !--full-path! flags control the prefix for the files in the archive. If the flag !--full-path! is set then the files will have the full source path as the prefix. If the flag !--prefix=! is set then the files will have !! as prefix. It's possible to create invalid file names with !--prefix=! so use with caution. Flag !--prefix! has priority over !--full-path!. Given a directory !/sourcedir! with the following: file1.txt dir1/file2.txt Running the command !rclone archive create /sourcedir /dest.tar.gz! will make an archive with the contents: file1.txt dir1/ dir1/file2.txt Running the command !rclone archive create --full-path /sourcedir /dest.tar.gz! will make an archive with the contents: sourcedir/file1.txt sourcedir/dir1/ sourcedir/dir1/file2.txt Running the command !rclone archive create --prefix=my_new_path /sourcedir /dest.tar.gz! will make an archive with the contents: my_new_path/file1.txt my_new_path/dir1/ my_new_path/dir1/file2.txt `, "!", "`"), Annotations: map[string]string{ "versionIntroduced": "v1.72", }, RunE: func(command *cobra.Command, args []string) error { var src, dst fs.Fs var dstFile string if len(args) == 1 { // source only, archive to stdout src = cmd.NewFsSrc(args) } else if len(args) == 2 { src = cmd.NewFsSrc(args) dst, dstFile = cmd.NewFsDstFile(args[1:2]) } else { cmd.CheckArgs(1, 2, command, args) } cmd.Run(false, false, command, func() error { fmt.Printf("dst=%v, dstFile=%q, src=%v, format=%q, prefix=%q\n", dst, dstFile, src, format, prefix) if prefix != "" { return ArchiveCreate(context.Background(), dst, dstFile, src, format, prefix) } else if fullPath { return ArchiveCreate(context.Background(), dst, dstFile, src, format, src.Root()) } return ArchiveCreate(context.Background(), dst, dstFile, src, format, "") }) return nil }, } // Globals var ( archiveFormats = map[string]archives.CompressedArchive{ "zip": archives.CompressedArchive{ Archival: archives.Zip{ContinueOnError: true}, }, "tar": archives.CompressedArchive{ Archival: archives.Tar{ContinueOnError: true}, }, "tar.gz": archives.CompressedArchive{ Compression: archives.Gz{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.bz2": archives.CompressedArchive{ Compression: archives.Bz2{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.lz": archives.CompressedArchive{ Compression: archives.Lzip{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.lz4": archives.CompressedArchive{ Compression: archives.Lz4{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.xz": archives.CompressedArchive{ Compression: archives.Xz{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.zst": archives.CompressedArchive{ Compression: archives.Zstd{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.br": archives.CompressedArchive{ Compression: archives.Brotli{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.sz": archives.CompressedArchive{ Compression: archives.Sz{}, Archival: archives.Tar{ContinueOnError: true}, }, "tar.mz": archives.CompressedArchive{ Compression: archives.MinLZ{}, Archival: archives.Tar{ContinueOnError: true}, }, } archiveExtensions = map[string]string{ // zip "*.zip": "zip", // tar "*.tar": "tar", // tar.gz "*.tar.gz": "tar.gz", "*.tgz": "tar.gz", "*.taz": "tar.gz", // tar.bz2 "*.tar.bz2": "tar.bz2", "*.tb2": "tar.bz2", "*.tbz": "tar.bz2", "*.tbz2": "tar.bz2", "*.tz2": "tar.bz2", // tar.lz "*.tar.lz": "tar.lz", // tar.lz4 "*.tar.lz4": "tar.lz4", // tar.xz "*.tar.xz": "tar.xz", "*.txz": "tar.xz", // tar.zst "*.tar.zst": "tar.zst", "*.tzst": "tar.zst", // tar.br "*.tar.br": "tar.br", // tar.sz "*.tar.sz": "tar.sz", // tar.mz "*.tar.mz": "tar.mz", } ) // sorted FileInfo list type archivesFileInfoList []archives.FileInfo func (a archivesFileInfoList) Len() int { return len(a) } func (a archivesFileInfoList) Less(i, j int) bool { if a[i].FileInfo.IsDir() == a[j].FileInfo.IsDir() { // both are same type, order by name return strings.Compare(a[i].NameInArchive, a[j].NameInArchive) < 0 } else if a[i].FileInfo.IsDir() { return strings.Compare(strings.TrimSuffix(a[i].NameInArchive, "/"), path.Dir(a[j].NameInArchive)) < 0 } return strings.Compare(path.Dir(a[i].NameInArchive), strings.TrimSuffix(a[j].NameInArchive, "/")) < 0 } func (a archivesFileInfoList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func getCompressor(format string, filename string) (archives.CompressedArchive, error) { var compressor archives.CompressedArchive var found bool // make filename lowercase for checks filename = strings.ToLower(filename) if format == "" { // format flag not set, get format from the file extension for pattern, formatName := range archiveExtensions { ok, err := path.Match(pattern, filename) if err != nil { // error in pattern return archives.CompressedArchive{}, fmt.Errorf("invalid extension pattern '%s'", pattern) } else if ok { // pattern matches filename, get compressor compressor, found = archiveFormats[formatName] break } } } else { // format flag set, look for it compressor, found = archiveFormats[format] } if found { return compressor, nil } else if format == "" { return archives.CompressedArchive{}, fmt.Errorf("format not set and can't be guessed from extension") } return archives.CompressedArchive{}, fmt.Errorf("invalid format '%s'", format) } // CheckValidDestination - takes (dst, dstFile) and checks it is valid func CheckValidDestination(ctx context.Context, dst fs.Fs, dstFile string) error { var err error // check if dst + dstFile is a file _, err = dst.NewObject(ctx, dstFile) if err == nil { // (dst, dstFile) is a valid file we can overwrite return nil } else if errors.Is(err, fs.ErrorIsDir) { // dst is a directory return fmt.Errorf("destination must not be a directory: %w", err) } else if !errors.Is(err, fs.ErrorObjectNotFound) { // dst is a directory (we need a filename) or some other error happened // not good, leave return fmt.Errorf("error reading destination: %w", err) } // if we are here dst points to a non existent path return nil } func loadMetadata(ctx context.Context, o fs.DirEntry) fs.Metadata { meta, err := fs.GetMetadata(ctx, o) if err != nil { meta = make(fs.Metadata, 0) } return meta } // ArchiveCreate - compresses/archive source to destination func ArchiveCreate(ctx context.Context, dst fs.Fs, dstFile string, src fs.Fs, format string, prefix string) error { var err error var list archivesFileInfoList var compArchive archives.CompressedArchive var totalLength int64 // check id dst is valid err = CheckValidDestination(ctx, dst, dstFile) if err != nil { return err } ci := fs.GetConfig(ctx) fi := filter.GetConfig(ctx) // get archive format compArchive, err = getCompressor(format, dstFile) if err != nil { return err } // get source files err = walk.ListR(ctx, src, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error { // get directories entries.ForDir(func(o fs.Directory) { var metadata fs.Metadata if ci.Metadata { metadata = loadMetadata(ctx, o) } if fi.Include(o.Remote(), o.Size(), o.ModTime(ctx), metadata) { info := files.NewArchiveFileInfo(ctx, o, prefix, metadata) list = append(list, info) } }) // get files entries.ForObject(func(o fs.Object) { var metadata fs.Metadata if ci.Metadata { metadata = loadMetadata(ctx, o) } if fi.Include(o.Remote(), o.Size(), o.ModTime(ctx), metadata) { info := files.NewArchiveFileInfo(ctx, o, prefix, metadata) list = append(list, info) totalLength += o.Size() } }) return nil }) if err != nil { return err } else if list.Len() == 0 { return fmt.Errorf("no files found in source") } sort.Stable(list) // create archive if ci.DryRun { // write nowhere counter := files.NewCountWriter(nil) err = compArchive.Archive(ctx, counter, list) // log totals fs.Infof(nil, "Total files added %d", list.Len()) fs.Infof(nil, "Total bytes read %d", totalLength) fs.Infof(nil, "Compressed file size %d", counter.Count()) return err } else if dst == nil { // write to stdout counter := files.NewCountWriter(os.Stdout) err = compArchive.Archive(ctx, counter, list) // log totals fs.Infof(nil, "Total files added %d", list.Len()) fs.Infof(nil, "Total bytes read %d", totalLength) fs.Infof(nil, "Compressed file size %d", counter.Count()) return err } // write to remote pipeReader, pipeWriter := io.Pipe() // write to pipewriter in background counter := files.NewCountWriter(pipeWriter) go func() { err := compArchive.Archive(ctx, counter, list) pipeWriter.CloseWithError(err) }() // rcat to remote from pipereader _, err = operations.Rcat(ctx, dst, dstFile, pipeReader, time.Now(), nil) // log totals fs.Infof(nil, "Total files added %d", list.Len()) fs.Infof(nil, "Total bytes read %d", totalLength) fs.Infof(nil, "Compressed file size %d", counter.Count()) return err }