mirror of
https://github.com/rclone/rclone.git
synced 2025-12-12 06:24:14 +01:00
34a20555ca
lib/transform adds the transform library, supporting advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations. It also adds the --name-transform flag for use with sync, copy, and move. Multiple transformations can be used in sequence, applied in the order they are specified on the command line. By default --name-transform will only apply to file names. The means only the leaf file name will be transformed. However some of the transforms would be better applied to the whole path or just directories. To choose which which part of the file path is affected some tags can be added to the --name-transform: file Only transform the leaf name of files (DEFAULT) dir Only transform name of directories - these may appear anywhere in the path all Transform the entire path for files and directories Example syntax: --name-transform file,prefix=ABC --name-transform dir,prefix=DEF
458 lines
14 KiB
Go
458 lines
14 KiB
Go
// Test transform
|
|
|
|
package sync
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
_ "github.com/rclone/rclone/backend/all"
|
|
"github.com/rclone/rclone/fs"
|
|
"github.com/rclone/rclone/fs/filter"
|
|
"github.com/rclone/rclone/fs/operations"
|
|
"github.com/rclone/rclone/fs/walk"
|
|
"github.com/rclone/rclone/fstest"
|
|
"github.com/rclone/rclone/lib/transform"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/text/unicode/norm"
|
|
)
|
|
|
|
var debug = ``
|
|
|
|
func TestTransform(t *testing.T) {
|
|
type args struct {
|
|
TransformOpt []string
|
|
TransformBackOpt []string
|
|
Lossless bool // whether the TransformBackAlgo is always losslessly invertible
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
}{
|
|
{name: "NFC", args: args{
|
|
TransformOpt: []string{"nfc"},
|
|
TransformBackOpt: []string{"nfd"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "NFD", args: args{
|
|
TransformOpt: []string{"nfd"},
|
|
TransformBackOpt: []string{"nfc"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "base64", args: args{
|
|
TransformOpt: []string{"base64encode"},
|
|
TransformBackOpt: []string{"base64encode"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "prefix", args: args{
|
|
TransformOpt: []string{"prefix=PREFIX"},
|
|
TransformBackOpt: []string{"trimprefix=PREFIX"},
|
|
Lossless: true,
|
|
}},
|
|
{name: "suffix", args: args{
|
|
TransformOpt: []string{"suffix=SUFFIX"},
|
|
TransformBackOpt: []string{"trimsuffix=SUFFIX"},
|
|
Lossless: true,
|
|
}},
|
|
{name: "truncate", args: args{
|
|
TransformOpt: []string{"truncate=10"},
|
|
TransformBackOpt: []string{"truncate=10"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "encoder", args: args{
|
|
TransformOpt: []string{"encoder=Colon,SquareBracket"},
|
|
TransformBackOpt: []string{"decoder=Colon,SquareBracket"},
|
|
Lossless: true,
|
|
}},
|
|
{name: "ISO-8859-1", args: args{
|
|
TransformOpt: []string{"ISO-8859-1"},
|
|
TransformBackOpt: []string{"ISO-8859-1"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "charmap", args: args{
|
|
TransformOpt: []string{"all,charmap=ISO-8859-7"},
|
|
TransformBackOpt: []string{"all,charmap=ISO-8859-7"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "lowercase", args: args{
|
|
TransformOpt: []string{"all,lowercase"},
|
|
TransformBackOpt: []string{"all,lowercase"},
|
|
Lossless: false,
|
|
}},
|
|
{name: "ascii", args: args{
|
|
TransformOpt: []string{"all,ascii"},
|
|
TransformBackOpt: []string{"all,ascii"},
|
|
Lossless: false,
|
|
}},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
r := fstest.NewRun(t)
|
|
defer r.Finalise()
|
|
|
|
r.Mkdir(context.Background(), r.Flocal)
|
|
r.Mkdir(context.Background(), r.Fremote)
|
|
items := makeTestFiles(t, r, "dir1")
|
|
deleteDSStore(t, r)
|
|
r.CheckRemoteListing(t, items, nil)
|
|
r.CheckLocalListing(t, items, nil)
|
|
|
|
err := transform.SetOptions(context.Background(), tt.args.TransformOpt...)
|
|
require.NoError(t, err)
|
|
|
|
err = Sync(context.Background(), r.Fremote, r.Flocal, true)
|
|
assert.NoError(t, err)
|
|
compareNames(t, r, items)
|
|
|
|
err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...)
|
|
require.NoError(t, err)
|
|
err = Sync(context.Background(), r.Fremote, r.Flocal, true)
|
|
assert.NoError(t, err)
|
|
compareNames(t, r, items)
|
|
|
|
if tt.args.Lossless {
|
|
deleteDSStore(t, r)
|
|
r.CheckRemoteItems(t, items...)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
const alphabet = "abcdefg123456789"
|
|
|
|
var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}
|
|
|
|
func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {
|
|
t.Helper()
|
|
n := 0
|
|
// Create test files
|
|
items := []fstest.Item{}
|
|
for _, c := range alphabet {
|
|
var out strings.Builder
|
|
for i := rune(0); i < 7; i++ {
|
|
out.WriteRune(c + i)
|
|
}
|
|
fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
|
|
fileName = strings.ToValidUTF8(fileName, "")
|
|
|
|
if debug != "" {
|
|
fileName = debug
|
|
}
|
|
|
|
item := r.WriteObject(context.Background(), fileName, fileName, t1)
|
|
r.WriteFile(fileName, fileName, t1)
|
|
items = append(items, item)
|
|
n++
|
|
|
|
if debug != "" {
|
|
break
|
|
}
|
|
}
|
|
|
|
for _, extra := range extras {
|
|
item := r.WriteObject(context.Background(), extra, extra, t1)
|
|
r.WriteFile(extra, extra, t1)
|
|
items = append(items, item)
|
|
}
|
|
|
|
return items
|
|
}
|
|
|
|
func deleteDSStore(t *testing.T, r *fstest.Run) {
|
|
ctxDSStore, fi := filter.AddConfig(context.Background())
|
|
err := fi.AddRule(`+ *.DS_Store`)
|
|
assert.NoError(t, err)
|
|
err = fi.AddRule(`- **`)
|
|
assert.NoError(t, err)
|
|
err = operations.Delete(ctxDSStore, r.Fremote)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
|
|
var entries fs.DirEntries
|
|
|
|
deleteDSStore(t, r)
|
|
err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error {
|
|
entries = append(entries, e...)
|
|
return nil
|
|
})
|
|
assert.NoError(t, err)
|
|
entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files
|
|
if strings.Contains(E.Remote(), ".DS_Store") {
|
|
err := operations.DeleteFile(context.Background(), E.(fs.Object))
|
|
assert.NoError(t, err)
|
|
return true
|
|
}
|
|
return false
|
|
})
|
|
require.Equal(t, len(items), entries.Len())
|
|
|
|
// sort by CONVERTED name
|
|
slices.SortStableFunc(items, func(a, b fstest.Item) int {
|
|
aConv := transform.Path(a.Path, false)
|
|
bConv := transform.Path(b.Path, false)
|
|
return cmp.Compare(aConv, bConv)
|
|
})
|
|
slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
|
|
return cmp.Compare(a.Remote(), b.Remote())
|
|
})
|
|
|
|
for i, e := range entries {
|
|
expect := transform.Path(items[i].Path, false)
|
|
msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))
|
|
assert.Equal(t, expect, e.Remote(), msg)
|
|
}
|
|
}
|
|
|
|
func detectEncoding(s string) string {
|
|
if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
|
|
return "BOTH"
|
|
}
|
|
if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
|
|
return "NFD"
|
|
}
|
|
if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {
|
|
return "NFC"
|
|
}
|
|
return "OTHER"
|
|
}
|
|
|
|
func TestTransformCopy(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "all,suffix_keep_extension=_somesuffix")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("sub dir/hello world.txt", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("sub dir_somesuffix/hello world_somesuffix.txt", "hello world", t1))
|
|
}
|
|
|
|
func TestDoubleTransform(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe", "hello world", t1))
|
|
}
|
|
|
|
func TestFileTag(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "file,prefix=tac", "file,prefix=tic")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))
|
|
}
|
|
|
|
func TestNoTag(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "prefix=tac", "prefix=tic")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))
|
|
}
|
|
|
|
func TestDirTag(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")
|
|
require.NoError(t, err)
|
|
r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
|
_, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
|
|
require.NoError(t, err)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})
|
|
r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
|
|
}
|
|
|
|
func TestAllTag(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
|
|
require.NoError(t, err)
|
|
r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
|
_, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
|
|
require.NoError(t, err)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})
|
|
r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
|
|
err = operations.Check(ctx, &operations.CheckOpt{Fsrc: r.Flocal, Fdst: r.Fremote}) // should not error even though dst has transformed names
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
func TestRunTwice(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))
|
|
|
|
// result should not change second time, since src is unchanged
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))
|
|
}
|
|
|
|
func TestSyntax(t *testing.T) {
|
|
ctx := context.Background()
|
|
err := transform.SetOptions(ctx, "prefix")
|
|
assert.Error(t, err) // should error as required value is missing
|
|
|
|
err = transform.SetOptions(ctx, "banana")
|
|
assert.Error(t, err) // should error as unrecognized option
|
|
|
|
err = transform.SetOptions(ctx, "=123")
|
|
assert.Error(t, err) // should error as required key is missing
|
|
|
|
err = transform.SetOptions(ctx, "prefix=123")
|
|
assert.NoError(t, err) // should not error
|
|
}
|
|
|
|
func TestConflicting(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "prefix=tac", "trimprefix=tac")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
// should result in no change as prefix and trimprefix cancel out
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("toe/toe/toe", "hello world", t1))
|
|
}
|
|
|
|
func TestMove(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
|
|
require.NoError(t, err)
|
|
r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
|
_, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
|
|
require.NoError(t, err)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = MoveDir(ctx, r.Fremote, r.Flocal, true, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalListing(t, []fstest.Item{}, []string{})
|
|
r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
|
|
}
|
|
|
|
func TestBase64(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "all,base64encode")
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))
|
|
|
|
// round trip
|
|
err = transform.SetOptions(ctx, "all,base64decode")
|
|
require.NoError(t, err)
|
|
ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Flocal, r.Fremote, true)
|
|
testLoggerVsLsf(ctx, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t)
|
|
require.NoError(t, err)
|
|
|
|
r.CheckLocalItems(t, file1)
|
|
r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))
|
|
}
|
|
|
|
func TestError(t *testing.T) {
|
|
ctx := context.Background()
|
|
r := fstest.NewRun(t)
|
|
err := transform.SetOptions(ctx, "all,prefix=ta/c") // has illegal character
|
|
require.NoError(t, err)
|
|
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
|
|
|
r.Mkdir(ctx, r.Fremote)
|
|
// ctx = predictDstFromLogger(ctx)
|
|
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
|
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
|
assert.Error(t, err)
|
|
|
|
r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})
|
|
r.CheckRemoteListing(t, []fstest.Item{}, []string{})
|
|
err = transform.SetOptions(ctx, "") // has illegal character
|
|
assert.NoError(t, err)
|
|
}
|