Add context propagation to rclone

- Change rclone/fs interfaces to accept context.Context
- Update interface implementations to use context.Context
- Change top level usage to propagate context to lover level functions

Context propagation is needed for stopping transfers and passing other
request-scoped values.

backend/http/http.go

@@ -5,6 +5,7 @@
 package http
 
 import (
+	"context"
 	"io"
 	"mime"
 	"net/http"
@@ -207,7 +208,7 @@ func (f *Fs) Precision() time.Duration {
 }
 
 // NewObject creates a new remote http file object
-func (f *Fs) NewObject(remote string) (fs.Object, error) {
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -359,7 +360,7 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 //
 // This should return ErrDirNotFound if the directory isn't
 // found.
-func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	if !strings.HasSuffix(dir, "/") && dir != "" {
 		dir += "/"
 	}
@@ -399,12 +400,12 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 // May create the object even if it returns an error - if so
 // will return the object and the error, otherwise will return
 // nil and the error
-func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	return nil, errorReadOnly
 }
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
-func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
 	return nil, errorReadOnly
 }
 
@@ -427,7 +428,7 @@ func (o *Object) Remote() string {
 }
 
 // Hash returns "" since HTTP (in Go or OpenSSH) doesn't support remote calculation of hashes
-func (o *Object) Hash(r hash.Type) (string, error) {
+func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	return "", hash.ErrUnsupported
 }
 
@@ -437,7 +438,7 @@ func (o *Object) Size() int64 {
 }
 
 // ModTime returns the modification time of the remote http file
-func (o *Object) ModTime() time.Time {
+func (o *Object) ModTime(ctx context.Context) time.Time {
 	return o.modTime
 }
 
@@ -480,7 +481,7 @@ func (o *Object) stat() error {
 // SetModTime sets the modification and access time to the specified time
 //
 // it also updates the info field
-func (o *Object) SetModTime(modTime time.Time) error {
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	return errorReadOnly
 }
 
@@ -490,7 +491,7 @@ func (o *Object) Storable() bool {
 }
 
 // Open a remote http file object for reading. Seek is supported
-func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	url := o.url()
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
@@ -517,27 +518,27 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 // Mkdir makes the root directory of the Fs object
-func (f *Fs) Mkdir(dir string) error {
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return errorReadOnly
 }
 
 // Remove a remote http file object
-func (o *Object) Remove() error {
+func (o *Object) Remove(ctx context.Context) error {
 	return errorReadOnly
 }
 
 // Rmdir removes the root directory of the Fs object
-func (f *Fs) Rmdir(dir string) error {
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return errorReadOnly
 }
 
 // Update in to the object with the modTime given of the given size
-func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	return errorReadOnly
 }
 
 // MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType() string {
+func (o *Object) MimeType(ctx context.Context) string {
 	return o.contentType
 }
 

backend/http/http_internal_test.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -64,7 +65,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 }
 
 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
-	entries, err := f.List("")
+	entries, err := f.List(context.Background(), "")
 	require.NoError(t, err)
 
 	sort.Sort(entries)
@@ -120,7 +121,7 @@ func TestListSubDir(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
 
-	entries, err := f.List("three")
+	entries, err := f.List(context.Background(), "three")
 	require.NoError(t, err)
 
 	sort.Sort(entries)
@@ -138,7 +139,7 @@ func TestNewObject(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
 
-	o, err := f.NewObject("four/under four.txt")
+	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
 
 	assert.Equal(t, "four/under four.txt", o.Remote())
@@ -148,7 +149,7 @@ func TestNewObject(t *testing.T) {
 
 	// Test the time is correct on the object
 
-	tObj := o.ModTime()
+	tObj := o.ModTime(context.Background())
 
 	fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
 	require.NoError(t, err)
@@ -158,7 +159,7 @@ func TestNewObject(t *testing.T) {
 	assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", o.Remote(), dt, time.Second, tObj, tFile, time.Second))
 
 	// check object not found
-	o, err = f.NewObject("not found.txt")
+	o, err = f.NewObject(context.Background(), "not found.txt")
 	assert.Nil(t, o)
 	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
@@ -167,11 +168,11 @@ func TestOpen(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
 
-	o, err := f.NewObject("four/under four.txt")
+	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
 
 	// Test normal read
-	fd, err := o.Open()
+	fd, err := o.Open(context.Background())
 	require.NoError(t, err)
 	data, err := ioutil.ReadAll(fd)
 	require.NoError(t, err)
@@ -179,7 +180,7 @@ func TestOpen(t *testing.T) {
 	assert.Equal(t, "beetroot\n", string(data))
 
 	// Test with range request
-	fd, err = o.Open(&fs.RangeOption{Start: 1, End: 5})
+	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
 	require.NoError(t, err)
 	data, err = ioutil.ReadAll(fd)
 	require.NoError(t, err)
@@ -191,12 +192,12 @@ func TestMimeType(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
 
-	o, err := f.NewObject("four/under four.txt")
+	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
 
 	do, ok := o.(fs.MimeTyper)
 	require.True(t, ok)
-	assert.Equal(t, "text/plain; charset=utf-8", do.MimeType())
+	assert.Equal(t, "text/plain; charset=utf-8", do.MimeType(context.Background()))
 }
 
 func TestIsAFileRoot(t *testing.T) {
@@ -216,7 +217,7 @@ func TestIsAFileSubDir(t *testing.T) {
 	f, err := NewFs(remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	entries, err := f.List("")
+	entries, err := f.List(context.Background(), "")
 	require.NoError(t, err)
 
 	sort.Sort(entries)