list: add ListDirSortedFn for callback oriented directory listing

This will be used for the out of memory sync

fs/list/list.go

@@ -39,8 +39,64 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entri
 	return filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
 }
 
-// filter (if required) and check the entries, then sort them
+// listP for every backend
-func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
+func listP(ctx context.Context, f fs.Fs, dir string, callback fs.ListRCallback) error {
+	if doListP := f.Features().ListP; doListP != nil {
+		return doListP(ctx, dir, callback)
+	}
+	// Fallback to List
+	entries, err := f.List(ctx, dir)
+	if err != nil {
+		return err
+	}
+	return callback(entries)
+}
+
+// DirSortedFn reads Object and *Dir into entries for the given Fs.
+//
+// dir is the start directory, "" for root
+//
+// If includeAll is specified all files will be added, otherwise only
+// files and directories passing the filter will be added.
+//
+// Files will be returned through callback in sorted order
+func DirSortedFn(ctx context.Context, f fs.Fs, includeAll bool, dir string, callback fs.ListRCallback, keyFn KeyFn) (err error) {
+	stats := accounting.Stats(ctx)
+	fi := filter.GetConfig(ctx)
+
+	// Sort the entries, in or out of memory
+	sorter, err := NewSorter(ctx, callback, keyFn)
+	if err != nil {
+		return fmt.Errorf("failed to create directory sorter: %w", err)
+	}
+	defer sorter.CleanUp()
+
+	// Get unfiltered entries from the fs
+	err = listP(ctx, f, dir, func(entries fs.DirEntries) error {
+		stats.Listed(int64(len(entries)))
+
+		// This should happen only if exclude files lives in the
+		// starting directory, otherwise ListDirSorted should not be
+		// called.
+		if !includeAll && fi.ListContainsExcludeFile(entries) {
+			fs.Debugf(dir, "Excluded")
+			return nil
+		}
+
+		entries, err := filterDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
+		if err != nil {
+			return err
+		}
+		return sorter.Add(entries)
+	})
+	if err != nil {
+		return err
+	}
+	return sorter.Send()
+}
+
+// Filter the entries passed in
+func filterDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
 	IncludeObject func(ctx context.Context, o fs.Object) bool,
 	IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
 	newEntries = entries[:0] // in place filter
@@ -96,7 +152,18 @@ func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll boo
 			newEntries = append(newEntries, entry)
 		}
 	}
-	entries = newEntries
+	return newEntries, nil
+}
+
+// filter and sort the entries
+func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
+	IncludeObject func(ctx context.Context, o fs.Object) bool,
+	IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
+	// Filter the directory entries (in place)
+	entries, err = filterDir(ctx, entries, includeAll, dir, IncludeObject, IncludeDirectory)
+	if err != nil {
+		return nil, err
+	}
 
 	// Sort the directory entries by Remote
 	//

fs/operations/listdirsorted_test.go

@@ -12,9 +12,9 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestListDirSorted is integration testing code in fs/list/list.go
+// testListDirSorted is integration testing code in fs/list/list.go
 // which can't be tested there due to import loops.
-func TestListDirSorted(t *testing.T) {
+func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) {
 	r := fstest.NewRun(t)
 
 	ctx := context.Background()
@@ -52,20 +52,20 @@ func TestListDirSorted(t *testing.T) {
 		return name
 	}
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, true, "")
+	items, err = listFn(context.Background(), r.Fremote, true, "")
 	require.NoError(t, err)
 	require.Len(t, items, 3)
 	assert.Equal(t, "a.txt", str(0))
 	assert.Equal(t, "sub dir/", str(1))
 	assert.Equal(t, "zend.txt", str(2))
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, false, "")
+	items, err = listFn(context.Background(), r.Fremote, false, "")
 	require.NoError(t, err)
 	require.Len(t, items, 2)
 	assert.Equal(t, "sub dir/", str(0))
 	assert.Equal(t, "zend.txt", str(1))
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir")
+	items, err = listFn(context.Background(), r.Fremote, true, "sub dir")
 	require.NoError(t, err)
 	require.Len(t, items, 4)
 	assert.Equal(t, "sub dir/hello world", str(0))
@@ -73,7 +73,7 @@ func TestListDirSorted(t *testing.T) {
 	assert.Equal(t, "sub dir/ignore dir/", str(2))
 	assert.Equal(t, "sub dir/sub sub dir/", str(3))
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
+	items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
 	require.NoError(t, err)
 	require.Len(t, items, 2)
 	assert.Equal(t, "sub dir/ignore dir/", str(0))
@@ -82,25 +82,45 @@ func TestListDirSorted(t *testing.T) {
 	// testing ignore file
 	fi.Opt.ExcludeFile = []string{".ignore"}
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
+	items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
 	require.NoError(t, err)
 	require.Len(t, items, 1)
 	assert.Equal(t, "sub dir/sub sub dir/", str(0))
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
+	items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
 	require.NoError(t, err)
 	require.Len(t, items, 0)
 
-	items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir/ignore dir")
+	items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir")
 	require.NoError(t, err)
 	require.Len(t, items, 2)
 	assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
 	assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
 
 	fi.Opt.ExcludeFile = nil
-	items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
+	items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
 	require.NoError(t, err)
 	require.Len(t, items, 2)
 	assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
 	assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
 }
+
+// TestListDirSorted is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func TestListDirSorted(t *testing.T) {
+	testListDirSorted(t, list.DirSorted)
+}
+
+// TestListDirSortedFn is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func TestListDirSortedFn(t *testing.T) {
+	listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
+		callback := func(newEntries fs.DirEntries) error {
+			entries = append(entries, newEntries...)
+			return nil
+		}
+		err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil)
+		return entries, err
+	}
+	testListDirSorted(t, listFn)
+}