johanels/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-11 22:14:05 +01:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

bisync: handle unicode and case normalization consistently - mostly-fixes #7270

Browse Source 
Before this change, Bisync sometimes normalized NFD to NFC and sometimes
did not, causing errors in some scenarios (particularly for users of macOS).
It was similarly inconsistent in its handling of case-insensitivity.

There were three main places where Bisync should have normalized, but didn't:

1. When building the list of files that need to be transferred during --resync
2. When building the list of deltas during a non-resync
3. When comparing Path1 to Path2 during --check-sync

After this change, 1 and 3 are resolved, and bisync supports
--no-unicode-normalization and --ignore-case-sync in the same way as sync.
2 will be addressed in a future update.
This commit is contained in:
nielash
2023-10-08 23:16:23 -04:00
parent 11afc3dde0
commit f7f4651828
32 changed files with 441 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
cmd/bisync/bisync_test.go
View File

@@ -98,10 +98,13 @@ var logHoppers = []string{
// subdirectories. The order inconsistency initially showed up in the
// listings and triggered reordering of log messages, but the actual
// files will in fact match.
`ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`,
`.* +.....Access test failed: Path[12] file not found in Path[12].*`,
// Test case `resync` suffered from the order of queued copies.
`(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`,
// Test case `normalization` can have random order of fix-case files.
`(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`,
}
// Some log lines can contain Windows path separator that must be
@@ -546,6 +549,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
case "copy-as":
b.checkArgs(args, 3, 3)
return b.copyFile(ctx, args[1], args[2], args[3])
case "copy-as-NFC":
b.checkArgs(args, 3, 3)
ci.NoUnicodeNormalization = true
ci.FixCase = true
return b.copyFile(ctx, args[1], norm.NFC.String(args[2]), norm.NFC.String(args[3]))
case "copy-as-NFD":
b.checkArgs(args, 3, 3)
ci.NoUnicodeNormalization = true
ci.FixCase = true
return b.copyFile(ctx, args[1], norm.NFD.String(args[2]), norm.NFD.String(args[3]))
case "copy-dir", "sync-dir":
b.checkArgs(args, 2, 2)
if fsrc, err = cache.Get(ctx, args[1]); err != nil {
@@ -565,6 +578,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
b.checkArgs(args, 1, 1)
return b.listSubdirs(ctx, args[1])
case "bisync":
ci.NoUnicodeNormalization = false
ci.IgnoreCaseSync = false
// ci.FixCase = true
return b.runBisync(ctx, args[1:])
case "test-func":
b.TestFn = testFunc
@@ -668,6 +684,16 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
fs2 = addSubdir(b.path2, val)
case "ignore-listing-checksum":
opt.IgnoreListingChecksum = true
case "no-norm":
ci.NoUnicodeNormalization = true
ci.IgnoreCaseSync = false
case "norm":
ci.NoUnicodeNormalization = false
ci.IgnoreCaseSync = true
case "fix-case":
ci.NoUnicodeNormalization = false
ci.IgnoreCaseSync = true
ci.FixCase = true
default:
return fmt.Errorf("invalid bisync option %q", arg)
}