johanels/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-11 22:14:05 +01:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

fs: Add --max-delete-size a delete size threshold

Browse Source 
Fixes #3329
This commit is contained in:
Leandro Sacchet
2022-08-03 11:53:02 -03:00
committed by Nick Craig-Wood
parent fb4600f6f9
commit f689db4422
6 changed files with 83 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
fs/operations/operations.go
View File

@@ -637,6 +637,12 @@ func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs
defer func() {
tr.Done(ctx, err)
}()
deletesSize := accounting.Stats(ctx).DeletesSize(0) // file not yet deleted, we should not add at this time
size := dst.Size()
if int64(ci.MaxDeleteSize) != -1 && (deletesSize+size) > int64(ci.MaxDeleteSize) {
return fserrors.FatalError(errors.New("--max-delete-size threshold reached"))
}
_ = accounting.Stats(ctx).DeletesSize(size) // here we count
numDeletes := accounting.Stats(ctx).Deletes(1)
if ci.MaxDelete != -1 && numDeletes > ci.MaxDelete {
return fserrors.FatalError(errors.New("--max-delete threshold reached"))

56
fs/operations/operations_test.go
View File

@@ -419,6 +419,62 @@ func TestDelete(t *testing.T) {
r.CheckRemoteItems(t, file3)
}
func TestMaxDelete(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
accounting.GlobalStats().ResetCounters()
ci.MaxDelete = 2
defer r.Finalise()
file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
r.CheckRemoteItems(t, file1, file2, file3)
err := operations.Delete(ctx, r.Fremote)
require.Error(t, err)
objects, _, _, err := operations.Count(ctx, r.Fremote)
require.NoError(t, err)
assert.Equal(t, int64(1), objects)
}
// TestMaxDeleteSizeLargeFile one of the files is larger than allowed
func TestMaxDeleteSizeLargeFile(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
accounting.GlobalStats().ResetCounters()
ci.MaxDeleteSize = 70
defer r.Finalise()
file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
r.CheckRemoteItems(t, file1, file2, file3)
err := operations.Delete(ctx, r.Fremote)
require.Error(t, err)
r.CheckRemoteItems(t, file3)
}
func TestMaxDeleteSize(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
r := fstest.NewRun(t)
accounting.GlobalStats().ResetCounters()
ci.MaxDeleteSize = 160
defer r.Finalise()
file1 := r.WriteObject(ctx, "small", "1234567890", t2) // 10 bytes
file2 := r.WriteObject(ctx, "medium", "------------------------------------------------------------", t1) // 60 bytes
file3 := r.WriteObject(ctx, "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
r.CheckRemoteItems(t, file1, file2, file3)
err := operations.Delete(ctx, r.Fremote)
require.Error(t, err)
objects, _, _, err := operations.Count(ctx, r.Fremote)
require.NoError(t, err)
assert.Equal(t, int64(1), objects) // 10 or 100 bytes
}
func TestRetry(t *testing.T) {
ctx := context.Background()