Add context propagation to rclone

- Change rclone/fs interfaces to accept context.Context
- Update interface implementations to use context.Context
- Change top level usage to propagate context to lover level functions

Context propagation is needed for stopping transfers and passing other
request-scoped values.

backend/cache/cache.go

@@ -509,7 +509,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	if doChangeNotify := wrappedFs.Features().ChangeNotify; doChangeNotify != nil {
 		pollInterval := make(chan time.Duration, 1)
 		pollInterval <- time.Duration(f.opt.ChunkCleanInterval)
-		doChangeNotify(f.receiveChangeNotify, pollInterval)
+		doChangeNotify(context.Background(), f.receiveChangeNotify, pollInterval)
 	}
 
 	f.features = (&fs.Features{
@@ -600,7 +600,7 @@ is used on top of the cache.
 	return f, fsErr
 }
 
-func (f *Fs) httpStats(in rc.Params) (out rc.Params, err error) {
+func (f *Fs) httpStats(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	out = make(rc.Params)
 	m, err := f.Stats()
 	if err != nil {
@@ -627,7 +627,7 @@ func (f *Fs) unwrapRemote(remote string) string {
 	return remote
 }
 
-func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
+func (f *Fs) httpExpireRemote(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	out = make(rc.Params)
 	remoteInt, ok := in["remote"]
 	if !ok {
@@ -672,7 +672,7 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 	return out, nil
 }
 
-func (f *Fs) rcFetch(in rc.Params) (rc.Params, error) {
+func (f *Fs) rcFetch(ctx context.Context, in rc.Params) (rc.Params, error) {
 	type chunkRange struct {
 		start, end int64
 	}
@@ -777,18 +777,18 @@ func (f *Fs) rcFetch(in rc.Params) (rc.Params, error) {
 	for _, pair := range files {
 		file, remote := pair[0], pair[1]
 		var status fileStatus
-		o, err := f.NewObject(remote)
+		o, err := f.NewObject(ctx, remote)
 		if err != nil {
 			fetchedChunks[file] = fileStatus{Error: err.Error()}
 			continue
 		}
 		co := o.(*Object)
-		err = co.refreshFromSource(true)
+		err = co.refreshFromSource(ctx, true)
 		if err != nil {
 			fetchedChunks[file] = fileStatus{Error: err.Error()}
 			continue
 		}
-		handle := NewObjectHandle(co, f)
+		handle := NewObjectHandle(ctx, co, f)
 		handle.UseMemory = false
 		handle.scaleWorkers(1)
 		walkChunkRanges(crs, co.Size(), func(chunk int64) {
@@ -874,7 +874,7 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
 // ChangeNotify can subscribe multiple callers
 // this is coupled with the wrapped fs ChangeNotify (if it supports it)
 // and also notifies other caches (i.e VFS) to clear out whenever something changes
-func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
+func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
 	f.parentsForgetMu.Lock()
 	defer f.parentsForgetMu.Unlock()
 	fs.Debugf(f, "subscribing to ChangeNotify")
@@ -921,7 +921,7 @@ func (f *Fs) TempUploadWaitTime() time.Duration {
 }
 
 // NewObject finds the Object at remote.
-func (f *Fs) NewObject(remote string) (fs.Object, error) {
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	var err error
 
 	fs.Debugf(f, "new object '%s'", remote)
@@ -940,16 +940,16 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	// search for entry in source or temp fs
 	var obj fs.Object
 	if f.opt.TempWritePath != "" {
-		obj, err = f.tempFs.NewObject(remote)
+		obj, err = f.tempFs.NewObject(ctx, remote)
 		// not found in temp fs
 		if err != nil {
 			fs.Debugf(remote, "find: not found in local cache fs")
-			obj, err = f.Fs.NewObject(remote)
+			obj, err = f.Fs.NewObject(ctx, remote)
 		} else {
 			fs.Debugf(obj, "find: found in local cache fs")
 		}
 	} else {
-		obj, err = f.Fs.NewObject(remote)
+		obj, err = f.Fs.NewObject(ctx, remote)
 	}
 
 	// not found in either fs
@@ -959,13 +959,13 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	}
 
 	// cache the new entry
-	co = ObjectFromOriginal(f, obj).persist()
+	co = ObjectFromOriginal(ctx, f, obj).persist()
 	fs.Debugf(co, "find: cached object")
 	return co, nil
 }
 
 // List the objects and directories in dir into entries
-func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	fs.Debugf(f, "list '%s'", dir)
 	cd := ShallowDirectory(f, dir)
 
@@ -995,12 +995,12 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			fs.Debugf(dir, "list: temp fs entries: %v", queuedEntries)
 
 			for _, queuedRemote := range queuedEntries {
-				queuedEntry, err := f.tempFs.NewObject(f.cleanRootFromPath(queuedRemote))
+				queuedEntry, err := f.tempFs.NewObject(ctx, f.cleanRootFromPath(queuedRemote))
 				if err != nil {
 					fs.Debugf(dir, "list: temp file not found in local fs: %v", err)
 					continue
 				}
-				co := ObjectFromOriginal(f, queuedEntry).persist()
+				co := ObjectFromOriginal(ctx, f, queuedEntry).persist()
 				fs.Debugf(co, "list: cached temp object")
 				cachedEntries = append(cachedEntries, co)
 			}
@@ -1008,7 +1008,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	}
 
 	// search from the source
-	sourceEntries, err := f.Fs.List(dir)
+	sourceEntries, err := f.Fs.List(ctx, dir)
 	if err != nil {
 		return nil, err
 	}
@@ -1046,11 +1046,11 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			if i < tmpCnt && cachedEntries[i].Remote() == oRemote {
 				continue
 			}
-			co := ObjectFromOriginal(f, o).persist()
+			co := ObjectFromOriginal(ctx, f, o).persist()
 			cachedEntries = append(cachedEntries, co)
 			fs.Debugf(dir, "list: cached object: %v", co)
 		case fs.Directory:
-			cdd := DirectoryFromOriginal(f, o)
+			cdd := DirectoryFromOriginal(ctx, f, o)
 			// check if the dir isn't expired and add it in cache if it isn't
 			if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(time.Duration(f.opt.InfoAge))) {
 				batchDirectories = append(batchDirectories, cdd)
@@ -1080,8 +1080,8 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return cachedEntries, nil
 }
 
-func (f *Fs) recurse(dir string, list *walk.ListRHelper) error {
-	entries, err := f.List(dir)
+func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) error {
+	entries, err := f.List(ctx, dir)
 	if err != nil {
 		return err
 	}
@@ -1089,7 +1089,7 @@ func (f *Fs) recurse(dir string, list *walk.ListRHelper) error {
 	for i := 0; i < len(entries); i++ {
 		innerDir, ok := entries[i].(fs.Directory)
 		if ok {
-			err := f.recurse(innerDir.Remote(), list)
+			err := f.recurse(ctx, innerDir.Remote(), list)
 			if err != nil {
 				return err
 			}
@@ -1106,21 +1106,21 @@ func (f *Fs) recurse(dir string, list *walk.ListRHelper) error {
 
 // ListR lists the objects and directories of the Fs starting
 // from dir recursively into out.
-func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	fs.Debugf(f, "list recursively from '%s'", dir)
 
 	// we check if the source FS supports ListR
 	// if it does, we'll use that to get all the entries, cache them and return
 	do := f.Fs.Features().ListR
 	if do != nil {
-		return do(dir, func(entries fs.DirEntries) error {
+		return do(ctx, dir, func(entries fs.DirEntries) error {
 			// we got called back with a set of entries so let's cache them and call the original callback
 			for _, entry := range entries {
 				switch o := entry.(type) {
 				case fs.Object:
-					_ = f.cache.AddObject(ObjectFromOriginal(f, o))
+					_ = f.cache.AddObject(ObjectFromOriginal(ctx, f, o))
 				case fs.Directory:
-					_ = f.cache.AddDir(DirectoryFromOriginal(f, o))
+					_ = f.cache.AddDir(DirectoryFromOriginal(ctx, f, o))
 				default:
 					return errors.Errorf("Unknown object type %T", entry)
 				}
@@ -1133,7 +1133,7 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 
 	// if we're here, we're gonna do a standard recursive traversal and cache everything
 	list := walk.NewListRHelper(callback)
-	err = f.recurse(dir, list)
+	err = f.recurse(ctx, dir, list)
 	if err != nil {
 		return err
 	}
@@ -1142,9 +1142,9 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 }
 
 // Mkdir makes the directory (container, bucket)
-func (f *Fs) Mkdir(dir string) error {
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	fs.Debugf(f, "mkdir '%s'", dir)
-	err := f.Fs.Mkdir(dir)
+	err := f.Fs.Mkdir(ctx, dir)
 	if err != nil {
 		return err
 	}
@@ -1172,7 +1172,7 @@ func (f *Fs) Mkdir(dir string) error {
 }
 
 // Rmdir removes the directory (container, bucket) if empty
-func (f *Fs) Rmdir(dir string) error {
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	fs.Debugf(f, "rmdir '%s'", dir)
 
 	if f.opt.TempWritePath != "" {
@@ -1182,9 +1182,9 @@ func (f *Fs) Rmdir(dir string) error {
 
 		// we check if the source exists on the remote and make the same move on it too if it does
 		// otherwise, we skip this step
-		_, err := f.UnWrap().List(dir)
+		_, err := f.UnWrap().List(ctx, dir)
 		if err == nil {
-			err := f.Fs.Rmdir(dir)
+			err := f.Fs.Rmdir(ctx, dir)
 			if err != nil {
 				return err
 			}
@@ -1192,10 +1192,10 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.ListR(ctx, f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
-					co := ObjectFromOriginal(f, oo)
+					co := ObjectFromOriginal(ctx, f, oo)
 					queuedEntries = append(queuedEntries, co)
 				}
 			}
@@ -1212,7 +1212,7 @@ func (f *Fs) Rmdir(dir string) error {
 			}
 		}
 	} else {
-		err := f.Fs.Rmdir(dir)
+		err := f.Fs.Rmdir(ctx, dir)
 		if err != nil {
 			return err
 		}
@@ -1243,7 +1243,7 @@ func (f *Fs) Rmdir(dir string) error {
 
 // DirMove moves src, srcRemote to this remote at dstRemote
 // using server side move operations.
-func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
 	fs.Debugf(f, "move dir '%s'/'%s' -> '%s'/'%s'", src.Root(), srcRemote, f.Root(), dstRemote)
 
 	do := f.Fs.Features().DirMove
@@ -1265,8 +1265,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		f.backgroundRunner.pause()
 		defer f.backgroundRunner.play()
 
-		_, errInWrap := srcFs.UnWrap().List(srcRemote)
-		_, errInTemp := f.tempFs.List(srcRemote)
+		_, errInWrap := srcFs.UnWrap().List(ctx, srcRemote)
+		_, errInTemp := f.tempFs.List(ctx, srcRemote)
 		// not found in either fs
 		if errInWrap != nil && errInTemp != nil {
 			return fs.ErrorDirNotFound
@@ -1275,7 +1275,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		// we check if the source exists on the remote and make the same move on it too if it does
 		// otherwise, we skip this step
 		if errInWrap == nil {
-			err := do(srcFs.UnWrap(), srcRemote, dstRemote)
+			err := do(ctx, srcFs.UnWrap(), srcRemote, dstRemote)
 			if err != nil {
 				return err
 			}
@@ -1288,10 +1288,10 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err := walk.ListR(ctx, f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
-					co := ObjectFromOriginal(f, oo)
+					co := ObjectFromOriginal(ctx, f, oo)
 					queuedEntries = append(queuedEntries, co)
 					if co.tempFileStartedUpload() {
 						fs.Errorf(co, "can't move - upload has already started. need to finish that")
@@ -1312,16 +1312,16 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 			fs.Errorf(srcRemote, "dirmove: can't move dir in temp fs")
 			return fs.ErrorCantDirMove
 		}
-		err = do(f.tempFs, srcRemote, dstRemote)
+		err = do(ctx, f.tempFs, srcRemote, dstRemote)
 		if err != nil {
 			return err
 		}
-		err = f.cache.ReconcileTempUploads(f)
+		err = f.cache.ReconcileTempUploads(ctx, f)
 		if err != nil {
 			return err
 		}
 	} else {
-		err := do(srcFs.UnWrap(), srcRemote, dstRemote)
+		err := do(ctx, srcFs.UnWrap(), srcRemote, dstRemote)
 		if err != nil {
 			return err
 		}
@@ -1427,10 +1427,10 @@ func (f *Fs) cacheReader(u io.Reader, src fs.ObjectInfo, originalRead func(inn i
 	}
 }
 
-type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
+type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error)
 
 // put in to the remote path
-func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
 	var err error
 	var obj fs.Object
 
@@ -1441,7 +1441,7 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
 		_ = f.cache.ExpireDir(parentCd)
 		f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory)
 
-		obj, err = f.tempFs.Put(in, src, options...)
+		obj, err = f.tempFs.Put(ctx, in, src, options...)
 		if err != nil {
 			fs.Errorf(obj, "put: failed to upload in temp fs: %v", err)
 			return nil, err
@@ -1456,14 +1456,14 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
 		// if cache writes is enabled write it first through cache
 	} else if f.opt.StoreWrites {
 		f.cacheReader(in, src, func(inn io.Reader) {
-			obj, err = put(inn, src, options...)
+			obj, err = put(ctx, inn, src, options...)
 		})
 		if err == nil {
 			fs.Debugf(obj, "put: uploaded to remote fs and saved in cache")
 		}
 		// last option: save it directly in remote fs
 	} else {
-		obj, err = put(in, src, options...)
+		obj, err = put(ctx, in, src, options...)
 		if err == nil {
 			fs.Debugf(obj, "put: uploaded to remote fs")
 		}
@@ -1475,7 +1475,7 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
 	}
 
 	// cache the new file
-	cachedObj := ObjectFromOriginal(f, obj)
+	cachedObj := ObjectFromOriginal(ctx, f, obj)
 
 	// deleting cached chunks and info to be replaced with new ones
 	_ = f.cache.RemoveObject(cachedObj.abs())
@@ -1498,33 +1498,33 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
 }
 
 // Put in to the remote path with the modTime given of the given size
-func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	fs.Debugf(f, "put data at '%s'", src.Remote())
-	return f.put(in, src, options, f.Fs.Put)
+	return f.put(ctx, in, src, options, f.Fs.Put)
 }
 
 // PutUnchecked uploads the object
-func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	do := f.Fs.Features().PutUnchecked
 	if do == nil {
 		return nil, errors.New("can't PutUnchecked")
 	}
 	fs.Debugf(f, "put data unchecked in '%s'", src.Remote())
-	return f.put(in, src, options, do)
+	return f.put(ctx, in, src, options, do)
 }
 
 // PutStream uploads the object
-func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	do := f.Fs.Features().PutStream
 	if do == nil {
 		return nil, errors.New("can't PutStream")
 	}
 	fs.Debugf(f, "put data streaming in '%s'", src.Remote())
-	return f.put(in, src, options, do)
+	return f.put(ctx, in, src, options, do)
 }
 
 // Copy src to this remote using server side copy operations.
-func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	fs.Debugf(f, "copy obj '%s' -> '%s'", src, remote)
 
 	do := f.Fs.Features().Copy
@@ -1544,7 +1544,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		return nil, fs.ErrorCantCopy
 	}
 	// refresh from source or abort
-	if err := srcObj.refreshFromSource(false); err != nil {
+	if err := srcObj.refreshFromSource(ctx, false); err != nil {
 		fs.Errorf(f, "can't copy %v - %v", src, err)
 		return nil, fs.ErrorCantCopy
 	}
@@ -1563,7 +1563,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		}
 	}
 
-	obj, err := do(srcObj.Object, remote)
+	obj, err := do(ctx, srcObj.Object, remote)
 	if err != nil {
 		fs.Errorf(srcObj, "error moving in cache: %v", err)
 		return nil, err
@@ -1571,7 +1571,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	fs.Debugf(obj, "copy: file copied")
 
 	// persist new
-	co := ObjectFromOriginal(f, obj).persist()
+	co := ObjectFromOriginal(ctx, f, obj).persist()
 	fs.Debugf(co, "copy: added to cache")
 	// expire the destination path
 	parentCd := NewDirectory(f, cleanPath(path.Dir(co.Remote())))
@@ -1598,7 +1598,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 }
 
 // Move src to this remote using server side move operations.
-func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	fs.Debugf(f, "moving obj '%s' -> %s", src, remote)
 
 	// if source fs doesn't support move abort
@@ -1619,7 +1619,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		return nil, fs.ErrorCantMove
 	}
 	// refresh from source or abort
-	if err := srcObj.refreshFromSource(false); err != nil {
+	if err := srcObj.refreshFromSource(ctx, false); err != nil {
 		fs.Errorf(f, "can't move %v - %v", src, err)
 		return nil, fs.ErrorCantMove
 	}
@@ -1655,7 +1655,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		fs.Debugf(srcObj, "move: queued file moved to %v", remote)
 	}
 
-	obj, err := do(srcObj.Object, remote)
+	obj, err := do(ctx, srcObj.Object, remote)
 	if err != nil {
 		fs.Errorf(srcObj, "error moving: %v", err)
 		return nil, err
@@ -1680,7 +1680,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	// advertise to ChangeNotify if wrapped doesn't do that
 	f.notifyChangeUpstreamIfNeeded(parentCd.Remote(), fs.EntryDirectory)
 	// persist new
-	cachedObj := ObjectFromOriginal(f, obj).persist()
+	cachedObj := ObjectFromOriginal(ctx, f, obj).persist()
 	fs.Debugf(cachedObj, "move: added to cache")
 	// expire new parent
 	parentCd = NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote())))
@@ -1702,7 +1702,7 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 // Purge all files in the root and the root directory
-func (f *Fs) Purge() error {
+func (f *Fs) Purge(ctx context.Context) error {
 	fs.Infof(f, "purging cache")
 	f.cache.Purge()
 
@@ -1711,7 +1711,7 @@ func (f *Fs) Purge() error {
 		return nil
 	}
 
-	err := do()
+	err := do(ctx)
 	if err != nil {
 		return err
 	}
@@ -1720,7 +1720,7 @@ func (f *Fs) Purge() error {
 }
 
 // CleanUp the trash in the Fs
-func (f *Fs) CleanUp() error {
+func (f *Fs) CleanUp(ctx context.Context) error {
 	f.CleanUpCache(false)
 
 	do := f.Fs.Features().CleanUp
@@ -1728,16 +1728,16 @@ func (f *Fs) CleanUp() error {
 		return nil
 	}
 
-	return do()
+	return do(ctx)
 }
 
 // About gets quota information from the Fs
-func (f *Fs) About() (*fs.Usage, error) {
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	do := f.Fs.Features().About
 	if do == nil {
 		return nil, errors.New("About not supported")
 	}
-	return do()
+	return do(ctx)
 }
 
 // Stats returns stats about the cache storage

backend/cache/cache_internal_test.go

@@ -4,6 +4,7 @@ package cache_test
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	goflag "flag"
 	"fmt"
@@ -120,7 +121,7 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	require.NoError(t, err)
 	listRootInner, err := runInstance.list(t, rootFs, innerFolder)
 	require.NoError(t, err)
-	listInner, err := rootFs2.List("")
+	listInner, err := rootFs2.List(context.Background(), "")
 	require.NoError(t, err)
 
 	require.Len(t, listRoot, 1)
@@ -138,10 +139,10 @@ func TestInternalVfsCache(t *testing.T) {
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir("test")
+	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
 	runInstance.writeObjectString(t, rootFs, "test/second", "content")
-	_, err = rootFs.List("test")
+	_, err = rootFs.List(context.Background(), "test")
 	require.NoError(t, err)
 
 	testReader := runInstance.randomReader(t, testSize)
@@ -266,7 +267,7 @@ func TestInternalObjNotFound(t *testing.T) {
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	obj, err := rootFs.NewObject("404")
+	obj, err := rootFs.NewObject(context.Background(), "404")
 	require.Error(t, err)
 	require.Nil(t, obj)
 }
@@ -445,7 +446,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	require.NoError(t, err)
 	log.Printf("original size: %v", originalSize)
 
-	o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	expectedSize := int64(len([]byte("test content")))
 	var data2 []byte
@@ -457,7 +458,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 		data2 = []byte("test content")
 	}
 	objInfo := object.NewStaticObjectInfo(runInstance.encryptRemoteIfNeeded(t, "data.bin"), time.Now(), int64(len(data2)), true, nil, cfs.UnWrap())
-	err = o.Update(bytes.NewReader(data2), objInfo)
+	err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data2)), o.Size())
 	log.Printf("updated size: %v", len(data2))
@@ -503,9 +504,9 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	} else {
 		testData = []byte("test content")
 	}
-	_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test"))
-	_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/one"))
-	_ = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/second"))
+	_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test"))
+	_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/one"))
+	_ = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/second"))
 	srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
 
 	// list in mount
@@ -515,7 +516,7 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	require.NoError(t, err)
 
 	// move file
-	_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
+	_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
 	require.NoError(t, err)
 
 	err = runInstance.retryBlock(func() error {
@@ -589,9 +590,9 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	} else {
 		testData = []byte("test content")
 	}
-	err = rootFs.Mkdir("test")
+	err = rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir("test/one")
+	err = rootFs.Mkdir(context.Background(), "test/one")
 	require.NoError(t, err)
 	srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
 
@@ -608,7 +609,7 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	require.False(t, found)
 
 	// move file
-	_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
+	_, err = cfs.UnWrap().Features().Move(context.Background(), srcObj, dstName)
 	require.NoError(t, err)
 
 	err = runInstance.retryBlock(func() error {
@@ -670,23 +671,23 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
-	o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	wrappedTime := time.Now().Add(-1 * time.Hour)
-	err = o.SetModTime(wrappedTime)
+	err = o.SetModTime(context.Background(), wrappedTime)
 	require.NoError(t, err)
 
 	// get a new instance from the cache
-	co, err := rootFs.NewObject("data.bin")
+	co, err := rootFs.NewObject(context.Background(), "data.bin")
 	require.NoError(t, err)
-	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
+	require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
 
 	cfs.DirCacheFlush() // flush the cache
 
 	// get a new instance from the cache
-	co, err = rootFs.NewObject("data.bin")
+	co, err = rootFs.NewObject(context.Background(), "data.bin")
 	require.NoError(t, err)
-	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
+	require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
 }
 
 func TestInternalChangeSeenAfterRc(t *testing.T) {
@@ -713,19 +714,19 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
-	o, err := cfs.UnWrap().NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	wrappedTime := time.Now().Add(-1 * time.Hour)
-	err = o.SetModTime(wrappedTime)
+	err = o.SetModTime(context.Background(), wrappedTime)
 	require.NoError(t, err)
 
 	// get a new instance from the cache
-	co, err := rootFs.NewObject("data.bin")
+	co, err := rootFs.NewObject(context.Background(), "data.bin")
 	require.NoError(t, err)
-	require.NotEqual(t, o.ModTime().String(), co.ModTime().String())
+	require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
 
 	// Call the rc function
-	m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
+	m, err := cacheExpire.Fn(context.Background(), rc.Params{"remote": "data.bin"})
 	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
@@ -733,9 +734,9 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	require.Contains(t, m["message"], "cached file cleared")
 
 	// get a new instance from the cache
-	co, err = rootFs.NewObject("data.bin")
+	co, err = rootFs.NewObject(context.Background(), "data.bin")
 	require.NoError(t, err)
-	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
+	require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
 	_, err = runInstance.list(t, rootFs, "")
 	require.NoError(t, err)
 
@@ -749,7 +750,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	require.Len(t, li1, 1)
 
 	// Call the rc function
-	m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
+	m, err = cacheExpire.Fn(context.Background(), rc.Params{"remote": "/"})
 	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
@@ -794,7 +795,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 	// create some rand test data
 	testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
-	o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	o, err := cfs.NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
 	co, ok := o.(*cache.Object)
 	require.True(t, ok)
@@ -833,7 +834,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, l, 1)
 
-	err = cfs.UnWrap().Mkdir(runInstance.encryptRemoteIfNeeded(t, "test/third"))
+	err = cfs.UnWrap().Mkdir(context.Background(), runInstance.encryptRemoteIfNeeded(t, "test/third"))
 	require.NoError(t, err)
 
 	l, err = runInstance.list(t, rootFs, "test")
@@ -868,14 +869,14 @@ func TestInternalBug2117(t *testing.T) {
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
-	err = cfs.UnWrap().Mkdir("test")
+	err = cfs.UnWrap().Mkdir(context.Background(), "test")
 	require.NoError(t, err)
 	for i := 1; i <= 4; i++ {
-		err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d", i))
+		err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d", i))
 		require.NoError(t, err)
 
 		for j := 1; j <= 4; j++ {
-			err = cfs.UnWrap().Mkdir(fmt.Sprintf("test/dir%d/dir%d", i, j))
+			err = cfs.UnWrap().Mkdir(context.Background(), fmt.Sprintf("test/dir%d/dir%d", i, j))
 			require.NoError(t, err)
 
 			runInstance.writeObjectString(t, cfs.UnWrap(), fmt.Sprintf("test/dir%d/dir%d/test.txt", i, j), "test")
@@ -1080,10 +1081,10 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 
 	if purge {
-		_ = f.Features().Purge()
+		_ = f.Features().Purge(context.Background())
 		require.NoError(t, err)
 	}
-	err = f.Mkdir("")
+	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
 	if r.useMount && !r.isMounted {
 		r.mountFs(t, f)
@@ -1097,7 +1098,7 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
 		r.unmountFs(t, f)
 	}
 
-	err := f.Features().Purge()
+	err := f.Features().Purge(context.Background())
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
@@ -1199,7 +1200,7 @@ func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.Read
 func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object {
 	in := bytes.NewReader(data)
 	_ = r.writeObjectReader(t, f, remote, in)
-	o, err := f.NewObject(remote)
+	o, err := f.NewObject(context.Background(), remote)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data)), o.Size())
 	return o
@@ -1208,7 +1209,7 @@ func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte
 func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Reader) fs.Object {
 	modTime := time.Now()
 	objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f)
-	obj, err := f.Put(in, objInfo)
+	obj, err := f.Put(context.Background(), in, objInfo)
 	require.NoError(t, err)
 	if r.useMount {
 		r.vfs.WaitForWriters(10 * time.Second)
@@ -1228,18 +1229,18 @@ func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []b
 		err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600)
 		require.NoError(t, err)
 		r.vfs.WaitForWriters(10 * time.Second)
-		obj, err = f.NewObject(remote)
+		obj, err = f.NewObject(context.Background(), remote)
 	} else {
 		in1 := bytes.NewReader(data1)
 		in2 := bytes.NewReader(data2)
 		objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
 		objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
 
-		obj, err = f.Put(in1, objInfo1)
+		obj, err = f.Put(context.Background(), in1, objInfo1)
 		require.NoError(t, err)
-		obj, err = f.NewObject(remote)
+		obj, err = f.NewObject(context.Background(), remote)
 		require.NoError(t, err)
-		err = obj.Update(in2, objInfo2)
+		err = obj.Update(context.Background(), in2, objInfo2)
 	}
 	require.NoError(t, err)
 
@@ -1268,7 +1269,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 			return checkSample, err
 		}
 	} else {
-		co, err := f.NewObject(remote)
+		co, err := f.NewObject(context.Background(), remote)
 		if err != nil {
 			return checkSample, err
 		}
@@ -1283,7 +1284,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLengthCheck bool) []byte {
 	size := end - offset
 	checkSample := make([]byte, size)
-	reader, err := o.Open(&fs.SeekOption{Offset: offset})
+	reader, err := o.Open(context.Background(), &fs.SeekOption{Offset: offset})
 	require.NoError(t, err)
 	totalRead, err := io.ReadFull(reader, checkSample)
 	if (err == io.EOF || err == io.ErrUnexpectedEOF) && noLengthCheck {
@@ -1300,7 +1301,7 @@ func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) {
 	if r.useMount {
 		err = os.Mkdir(path.Join(r.mntDir, remote), 0700)
 	} else {
-		err = f.Mkdir(remote)
+		err = f.Mkdir(context.Background(), remote)
 	}
 	require.NoError(t, err)
 }
@@ -1312,11 +1313,11 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 		err = os.Remove(path.Join(r.mntDir, remote))
 	} else {
 		var obj fs.Object
-		obj, err = f.NewObject(remote)
+		obj, err = f.NewObject(context.Background(), remote)
 		if err != nil {
-			err = f.Rmdir(remote)
+			err = f.Rmdir(context.Background(), remote)
 		} else {
-			err = obj.Remove()
+			err = obj.Remove(context.Background())
 		}
 	}
 
@@ -1334,7 +1335,7 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 		}
 	} else {
 		var list fs.DirEntries
-		list, err = f.List(remote)
+		list, err = f.List(context.Background(), remote)
 		for _, ll := range list {
 			l = append(l, ll)
 		}
@@ -1353,7 +1354,7 @@ func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string {
 		}
 	} else {
 		var list fs.DirEntries
-		list, err = f.List(remote)
+		list, err = f.List(context.Background(), remote)
 		for _, ll := range list {
 			l = append(l, ll.Remote())
 		}
@@ -1393,7 +1394,7 @@ func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 		}
 		r.vfs.WaitForWriters(10 * time.Second)
 	} else if rootFs.Features().DirMove != nil {
-		err = rootFs.Features().DirMove(rootFs, src, dst)
+		err = rootFs.Features().DirMove(context.Background(), rootFs, src, dst)
 		if err != nil {
 			return err
 		}
@@ -1415,11 +1416,11 @@ func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
 		}
 		r.vfs.WaitForWriters(10 * time.Second)
 	} else if rootFs.Features().Move != nil {
-		obj1, err := rootFs.NewObject(src)
+		obj1, err := rootFs.NewObject(context.Background(), src)
 		if err != nil {
 			return err
 		}
-		_, err = rootFs.Features().Move(obj1, dst)
+		_, err = rootFs.Features().Move(context.Background(), obj1, dst)
 		if err != nil {
 			return err
 		}
@@ -1441,11 +1442,11 @@ func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
 		}
 		r.vfs.WaitForWriters(10 * time.Second)
 	} else if rootFs.Features().Copy != nil {
-		obj, err := rootFs.NewObject(src)
+		obj, err := rootFs.NewObject(context.Background(), src)
 		if err != nil {
 			return err
 		}
-		_, err = rootFs.Features().Copy(obj, dst)
+		_, err = rootFs.Features().Copy(context.Background(), obj, dst)
 		if err != nil {
 			return err
 		}
@@ -1467,11 +1468,11 @@ func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error)
 		}
 		return fi.ModTime(), nil
 	}
-	obj1, err := rootFs.NewObject(src)
+	obj1, err := rootFs.NewObject(context.Background(), src)
 	if err != nil {
 		return time.Time{}, err
 	}
-	return obj1.ModTime(), nil
+	return obj1.ModTime(context.Background()), nil
 }
 
 func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
@@ -1484,7 +1485,7 @@ func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
 		}
 		return fi.Size(), nil
 	}
-	obj1, err := rootFs.NewObject(src)
+	obj1, err := rootFs.NewObject(context.Background(), src)
 	if err != nil {
 		return int64(0), err
 	}
@@ -1507,14 +1508,14 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		_, err = f.WriteString(data + append)
 	} else {
 		var obj1 fs.Object
-		obj1, err = rootFs.NewObject(src)
+		obj1, err = rootFs.NewObject(context.Background(), src)
 		if err != nil {
 			return err
 		}
 		data1 := []byte(data + append)
 		r := bytes.NewReader(data1)
 		objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
-		err = obj1.Update(r, objInfo1)
+		err = obj1.Update(context.Background(), r, objInfo1)
 	}
 
 	return err

backend/cache/cache_upload_test.go

@@ -3,6 +3,7 @@
 package cache_test
 
 import (
+	"context"
 	"fmt"
 	"math/rand"
 	"os"
@@ -85,11 +86,11 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir("one")
+	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
-	err = rootFs.Mkdir("one/test")
+	err = rootFs.Mkdir(context.Background(), "one/test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir("second")
+	err = rootFs.Mkdir(context.Background(), "second")
 	require.NoError(t, err)
 
 	// create some rand test data
@@ -122,11 +123,11 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir("one")
+	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
-	err = rootFs.Mkdir("one/test")
+	err = rootFs.Mkdir(context.Background(), "one/test")
 	require.NoError(t, err)
-	err = rootFs.Mkdir("second")
+	err = rootFs.Mkdir(context.Background(), "second")
 	require.NoError(t, err)
 
 	// create some rand test data
@@ -165,7 +166,7 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
-	err := rootFs.Mkdir("test")
+	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
 	minSize := 5242880
 	maxSize := 10485760
@@ -233,9 +234,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.dirMove(t, rootFs, "test", "second")
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
+		_, err = rootFs.NewObject(context.Background(), "test/one")
 		require.Error(t, err)
-		_, err = rootFs.NewObject("second/one")
+		_, err = rootFs.NewObject(context.Background(), "second/one")
 		require.NoError(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -256,7 +257,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.rm(t, rootFs, "test")
 	require.Error(t, err)
 	require.Contains(t, err.Error(), "directory not empty")
-	_, err = rootFs.NewObject("test/one")
+	_, err = rootFs.NewObject(context.Background(), "test/one")
 	require.NoError(t, err)
 	// validate that it exists in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -270,9 +271,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	if err != errNotSupported {
 		require.NoError(t, err)
 		// try to read from it
-		_, err = rootFs.NewObject("test/one")
+		_, err = rootFs.NewObject(context.Background(), "test/one")
 		require.Error(t, err)
-		_, err = rootFs.NewObject("test/second")
+		_, err = rootFs.NewObject(context.Background(), "test/second")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -289,9 +290,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
+		_, err = rootFs.NewObject(context.Background(), "test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/third")
+		_, err = rootFs.NewObject(context.Background(), "test/third")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -306,7 +307,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	// test Remove -- allowed
 	err = runInstance.rm(t, rootFs, "test/one")
 	require.NoError(t, err)
-	_, err = rootFs.NewObject("test/one")
+	_, err = rootFs.NewObject(context.Background(), "test/one")
 	require.Error(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -318,7 +319,7 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 	require.NoError(t, err)
 	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
 	require.NoError(t, err)
-	obj2, err := rootFs.NewObject("test/one")
+	obj2, err := rootFs.NewObject(context.Background(), "test/one")
 	require.NoError(t, err)
 	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
 	require.Equal(t, "one content updated", string(data2))
@@ -366,7 +367,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	err = runInstance.dirMove(t, rootFs, "test", "second")
 	if err != errNotSupported {
 		require.Error(t, err)
-		_, err = rootFs.NewObject("test/one")
+		_, err = rootFs.NewObject(context.Background(), "test/one")
 		require.NoError(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -378,7 +379,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	// test Rmdir
 	err = runInstance.rm(t, rootFs, "test")
 	require.Error(t, err)
-	_, err = rootFs.NewObject("test/one")
+	_, err = rootFs.NewObject(context.Background(), "test/one")
 	require.NoError(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -389,9 +390,9 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	if err != errNotSupported {
 		require.Error(t, err)
 		// try to read from it
-		_, err = rootFs.NewObject("test/one")
+		_, err = rootFs.NewObject(context.Background(), "test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/second")
+		_, err = rootFs.NewObject(context.Background(), "test/second")
 		require.Error(t, err)
 		// validate that it exists in temp fs
 		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
@@ -404,9 +405,9 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
 	if err != errNotSupported {
 		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
+		_, err = rootFs.NewObject(context.Background(), "test/one")
 		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/third")
+		_, err = rootFs.NewObject(context.Background(), "test/third")
 		require.NoError(t, err)
 		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
 		require.NoError(t, err)
@@ -421,7 +422,7 @@ func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	// test Remove
 	err = runInstance.rm(t, rootFs, "test/one")
 	require.Error(t, err)
-	_, err = rootFs.NewObject("test/one")
+	_, err = rootFs.NewObject(context.Background(), "test/one")
 	require.NoError(t, err)
 	// validate that it doesn't exist in temp fs
 	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))

backend/cache/directory.go

@@ -3,6 +3,7 @@
 package cache
 
 import (
+	"context"
 	"path"
 	"time"
 
@@ -55,7 +56,7 @@ func ShallowDirectory(f *Fs, remote string) *Directory {
 }
 
 // DirectoryFromOriginal builds one from a generic fs.Directory
-func DirectoryFromOriginal(f *Fs, d fs.Directory) *Directory {
+func DirectoryFromOriginal(ctx context.Context, f *Fs, d fs.Directory) *Directory {
 	var cd *Directory
 	fullRemote := path.Join(f.Root(), d.Remote())
 
@@ -67,7 +68,7 @@ func DirectoryFromOriginal(f *Fs, d fs.Directory) *Directory {
 		CacheFs:      f,
 		Name:         name,
 		Dir:          dir,
-		CacheModTime: d.ModTime().UnixNano(),
+		CacheModTime: d.ModTime(ctx).UnixNano(),
 		CacheSize:    d.Size(),
 		CacheItems:   d.Items(),
 		CacheType:    "Directory",
@@ -110,7 +111,7 @@ func (d *Directory) parentRemote() string {
 }
 
 // ModTime returns the cached ModTime
-func (d *Directory) ModTime() time.Time {
+func (d *Directory) ModTime(ctx context.Context) time.Time {
 	return time.Unix(0, d.CacheModTime)
 }
 

backend/cache/handle.go

@@ -3,6 +3,7 @@
 package cache
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"path"
@@ -40,6 +41,7 @@ func initBackgroundUploader(fs *Fs) (*backgroundWriter, error) {
 
 // Handle is managing the read/write/seek operations on an open handle
 type Handle struct {
+	ctx            context.Context
 	cachedObject   *Object
 	cfs            *Fs
 	memory         *Memory
@@ -58,8 +60,9 @@ type Handle struct {
 }
 
 // NewObjectHandle returns a new Handle for an existing Object
-func NewObjectHandle(o *Object, cfs *Fs) *Handle {
+func NewObjectHandle(ctx context.Context, o *Object, cfs *Fs) *Handle {
 	r := &Handle{
+		ctx:           ctx,
 		cachedObject:  o,
 		cfs:           cfs,
 		offset:        0,
@@ -351,7 +354,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 	r := w.rc
 	if w.rc == nil {
 		r, err = w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) {
-			return w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1})
+			return w.r.cachedObject.Object.Open(w.r.ctx, &fs.RangeOption{Start: offset, End: end - 1})
 		})
 		if err != nil {
 			return nil, err
@@ -361,7 +364,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	if !closeOpen {
 		if do, ok := r.(fs.RangeSeeker); ok {
-			_, err = do.RangeSeek(offset, io.SeekStart, end-offset)
+			_, err = do.RangeSeek(w.r.ctx, offset, io.SeekStart, end-offset)
 			return r, err
 		} else if do, ok := r.(io.Seeker); ok {
 			_, err = do.Seek(offset, io.SeekStart)
@@ -371,7 +374,7 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	_ = w.rc.Close()
 	return w.r.cacheFs().openRateLimited(func() (io.ReadCloser, error) {
-		r, err = w.r.cachedObject.Object.Open(&fs.RangeOption{Start: offset, End: end - 1})
+		r, err = w.r.cachedObject.Object.Open(w.r.ctx, &fs.RangeOption{Start: offset, End: end - 1})
 		if err != nil {
 			return nil, err
 		}
@@ -449,7 +452,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	// we seem to be getting only errors so we abort
 	if err != nil {
 		fs.Errorf(w, "object open failed %v: %v", chunkStart, err)
-		err = w.r.cachedObject.refreshFromSource(true)
+		err = w.r.cachedObject.refreshFromSource(w.r.ctx, true)
 		if err != nil {
 			fs.Errorf(w, "%v", err)
 		}
@@ -462,7 +465,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	sourceRead, err = io.ReadFull(w.rc, data)
 	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err)
-		err = w.r.cachedObject.refreshFromSource(true)
+		err = w.r.cachedObject.refreshFromSource(w.r.ctx, true)
 		if err != nil {
 			fs.Errorf(w, "%v", err)
 		}
@@ -588,7 +591,7 @@ func (b *backgroundWriter) run() {
 		remote := b.fs.cleanRootFromPath(absPath)
 		b.notify(remote, BackgroundUploadStarted, nil)
 		fs.Infof(remote, "background upload: started upload")
-		err = operations.MoveFile(b.fs.UnWrap(), b.fs.tempFs, remote, remote)
+		err = operations.MoveFile(context.TODO(), b.fs.UnWrap(), b.fs.tempFs, remote, remote)
 		if err != nil {
 			b.notify(remote, BackgroundUploadError, err)
 			_ = b.fs.cache.rollbackPendingUpload(absPath)
@@ -598,14 +601,14 @@ func (b *backgroundWriter) run() {
 		// clean empty dirs up to root
 		thisDir := cleanPath(path.Dir(remote))
 		for thisDir != "" {
-			thisList, err := b.fs.tempFs.List(thisDir)
+			thisList, err := b.fs.tempFs.List(context.TODO(), thisDir)
 			if err != nil {
 				break
 			}
 			if len(thisList) > 0 {
 				break
 			}
-			err = b.fs.tempFs.Rmdir(thisDir)
+			err = b.fs.tempFs.Rmdir(context.TODO(), thisDir)
 			fs.Debugf(thisDir, "cleaned from temp path")
 			if err != nil {
 				break

backend/cache/object.go

@@ -3,6 +3,7 @@
 package cache
 
 import (
+	"context"
 	"io"
 	"path"
 	"sync"
@@ -68,7 +69,7 @@ func NewObject(f *Fs, remote string) *Object {
 }
 
 // ObjectFromOriginal builds one from a generic fs.Object
-func ObjectFromOriginal(f *Fs, o fs.Object) *Object {
+func ObjectFromOriginal(ctx context.Context, f *Fs, o fs.Object) *Object {
 	var co *Object
 	fullRemote := cleanPath(path.Join(f.Root(), o.Remote()))
 	dir, name := path.Split(fullRemote)
@@ -92,13 +93,13 @@ func ObjectFromOriginal(f *Fs, o fs.Object) *Object {
 		CacheType: cacheType,
 		CacheTs:   time.Now(),
 	}
-	co.updateData(o)
+	co.updateData(ctx, o)
 	return co
 }
 
-func (o *Object) updateData(source fs.Object) {
+func (o *Object) updateData(ctx context.Context, source fs.Object) {
 	o.Object = source
-	o.CacheModTime = source.ModTime().UnixNano()
+	o.CacheModTime = source.ModTime(ctx).UnixNano()
 	o.CacheSize = source.Size()
 	o.CacheStorable = source.Storable()
 	o.CacheTs = time.Now()
@@ -130,20 +131,20 @@ func (o *Object) abs() string {
 }
 
 // ModTime returns the cached ModTime
-func (o *Object) ModTime() time.Time {
-	_ = o.refresh()
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	_ = o.refresh(ctx)
 	return time.Unix(0, o.CacheModTime)
 }
 
 // Size returns the cached Size
 func (o *Object) Size() int64 {
-	_ = o.refresh()
+	_ = o.refresh(context.TODO())
 	return o.CacheSize
 }
 
 // Storable returns the cached Storable
 func (o *Object) Storable() bool {
-	_ = o.refresh()
+	_ = o.refresh(context.TODO())
 	return o.CacheStorable
 }
 
@@ -151,18 +152,18 @@ func (o *Object) Storable() bool {
 // all these conditions must be true to ignore a refresh
 // 1. cache ts didn't expire yet
 // 2. is not pending a notification from the wrapped fs
-func (o *Object) refresh() error {
+func (o *Object) refresh(ctx context.Context) error {
 	isNotified := o.CacheFs.isNotifiedRemote(o.Remote())
 	isExpired := time.Now().After(o.CacheTs.Add(time.Duration(o.CacheFs.opt.InfoAge)))
 	if !isExpired && !isNotified {
 		return nil
 	}
 
-	return o.refreshFromSource(true)
+	return o.refreshFromSource(ctx, true)
 }
 
 // refreshFromSource requests the original FS for the object in case it comes from a cached entry
-func (o *Object) refreshFromSource(force bool) error {
+func (o *Object) refreshFromSource(ctx context.Context, force bool) error {
 	o.refreshMutex.Lock()
 	defer o.refreshMutex.Unlock()
 	var err error
@@ -172,29 +173,29 @@ func (o *Object) refreshFromSource(force bool) error {
 		return nil
 	}
 	if o.isTempFile() {
-		liveObject, err = o.ParentFs.NewObject(o.Remote())
+		liveObject, err = o.ParentFs.NewObject(ctx, o.Remote())
 		err = errors.Wrapf(err, "in parent fs %v", o.ParentFs)
 	} else {
-		liveObject, err = o.CacheFs.Fs.NewObject(o.Remote())
+		liveObject, err = o.CacheFs.Fs.NewObject(ctx, o.Remote())
 		err = errors.Wrapf(err, "in cache fs %v", o.CacheFs.Fs)
 	}
 	if err != nil {
 		fs.Errorf(o, "error refreshing object in : %v", err)
 		return err
 	}
-	o.updateData(liveObject)
+	o.updateData(ctx, liveObject)
 	o.persist()
 
 	return nil
 }
 
 // SetModTime sets the ModTime of this object
-func (o *Object) SetModTime(t time.Time) error {
-	if err := o.refreshFromSource(false); err != nil {
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	if err := o.refreshFromSource(ctx, false); err != nil {
 		return err
 	}
 
-	err := o.Object.SetModTime(t)
+	err := o.Object.SetModTime(ctx, t)
 	if err != nil {
 		return err
 	}
@@ -207,19 +208,19 @@ func (o *Object) SetModTime(t time.Time) error {
 }
 
 // Open is used to request a specific part of the file using fs.RangeOption
-func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
 	var err error
 
 	if o.Object == nil {
-		err = o.refreshFromSource(true)
+		err = o.refreshFromSource(ctx, true)
 	} else {
-		err = o.refresh()
+		err = o.refresh(ctx)
 	}
 	if err != nil {
 		return nil, err
 	}
 
-	cacheReader := NewObjectHandle(o, o.CacheFs)
+	cacheReader := NewObjectHandle(ctx, o, o.CacheFs)
 	var offset, limit int64 = 0, -1
 	for _, option := range options {
 		switch x := option.(type) {
@@ -238,8 +239,8 @@ func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
 }
 
 // Update will change the object data
-func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if err := o.refreshFromSource(false); err != nil {
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	if err := o.refreshFromSource(ctx, false); err != nil {
 		return err
 	}
 	// pause background uploads if active
@@ -254,7 +255,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	fs.Debugf(o, "updating object contents with size %v", src.Size())
 
 	// FIXME use reliable upload
-	err := o.Object.Update(in, src, options...)
+	err := o.Object.Update(ctx, in, src, options...)
 	if err != nil {
 		fs.Errorf(o, "error updating source: %v", err)
 		return err
@@ -265,7 +266,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// advertise to ChangeNotify if wrapped doesn't do that
 	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)
 
-	o.CacheModTime = src.ModTime().UnixNano()
+	o.CacheModTime = src.ModTime(ctx).UnixNano()
 	o.CacheSize = src.Size()
 	o.CacheHashes = make(map[hash.Type]string)
 	o.CacheTs = time.Now()
@@ -275,8 +276,8 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 }
 
 // Remove deletes the object from both the cache and the source
-func (o *Object) Remove() error {
-	if err := o.refreshFromSource(false); err != nil {
+func (o *Object) Remove(ctx context.Context) error {
+	if err := o.refreshFromSource(ctx, false); err != nil {
 		return err
 	}
 	// pause background uploads if active
@@ -288,7 +289,7 @@ func (o *Object) Remove() error {
 			return errors.Errorf("%v is currently uploading, can't delete", o)
 		}
 	}
-	err := o.Object.Remove()
+	err := o.Object.Remove(ctx)
 	if err != nil {
 		return err
 	}
@@ -306,8 +307,8 @@ func (o *Object) Remove() error {
 
 // Hash requests a hash of the object and stores in the cache
 // since it might or might not be called, this is lazy loaded
-func (o *Object) Hash(ht hash.Type) (string, error) {
-	_ = o.refresh()
+func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
+	_ = o.refresh(ctx)
 	if o.CacheHashes == nil {
 		o.CacheHashes = make(map[hash.Type]string)
 	}
@@ -316,10 +317,10 @@ func (o *Object) Hash(ht hash.Type) (string, error) {
 	if found {
 		return cachedHash, nil
 	}
-	if err := o.refreshFromSource(false); err != nil {
+	if err := o.refreshFromSource(ctx, false); err != nil {
 		return "", err
 	}
-	liveHash, err := o.Object.Hash(ht)
+	liveHash, err := o.Object.Hash(ctx, ht)
 	if err != nil {
 		return "", err
 	}

backend/cache/storage_persistent.go

@@ -4,6 +4,7 @@ package cache
 
 import (
 	"bytes"
+	"context"
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
@@ -1014,7 +1015,7 @@ func (b *Persistent) SetPendingUploadToStarted(remote string) error {
 }
 
 // ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue
-func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
+func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
 		_ = tx.DeleteBucket([]byte(tempBucket))
 		bucket, err := tx.CreateBucketIfNotExists([]byte(tempBucket))
@@ -1023,7 +1024,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		err = walk.ListR(ctx, cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)