azureblob,b2,gcs,qingstor,s3,swift: Don't check for bucket/container presense if listing was OK

In a typical rclone copy to a bucket/container based remote, before
this change we were doing a list, followed by a HEAD of the bucket to
check it existed before doing the copy.  The fact the list succeeded
means the bucket exists so mark it OK at that point.

Issue #1421

backend/azureblob/azureblob.go

@@ -405,6 +405,16 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Blob, isDirectory boo
 	return o, nil
 }
 
+// mark the container as being OK
+func (f *Fs) markContainerOK() {
+	if f.container != "" {
+		f.containerOKMu.Lock()
+		f.containerOK = true
+		f.containerDeleted = false
+		f.containerOKMu.Unlock()
+	}
+}
+
 // listDir lists a single directory
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	err = f.list(dir, false, listChunkSize, func(remote string, object *storage.Blob, isDirectory bool) error {
@@ -420,6 +430,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, err
 	}
+	// container must be present if listing succeeded
+	f.markContainerOK()
 	return entries, nil
 }
 
@@ -491,6 +503,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	// container must be present if listing succeeded
+	f.markContainerOK()
 	return list.Flush()
 }
 

backend/b2/b2.go

@@ -551,6 +551,15 @@ func (f *Fs) itemToDirEntry(remote string, object *api.File, isDirectory bool, l
 	return o, nil
 }
 
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+	if f.bucket != "" {
+		f.bucketOKMu.Lock()
+		f.bucketOK = true
+		f.bucketOKMu.Unlock()
+	}
+}
+
 // listDir lists a single directory
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	last := ""
@@ -567,6 +576,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return entries, nil
 }
 
@@ -634,6 +645,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return list.Flush()
 }
 

backend/googlecloudstorage/googlecloudstorage.go

@@ -463,6 +463,15 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory b
 	return o, nil
 }
 
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+	if f.bucket != "" {
+		f.bucketOKMu.Lock()
+		f.bucketOK = true
+		f.bucketOKMu.Unlock()
+	}
+}
+
 // listDir lists a single directory
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	// List the objects
@@ -479,6 +488,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return entries, err
 }
 
@@ -555,6 +566,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return list.Flush()
 }
 

backend/qingstor/qingstor.go

@@ -519,6 +519,16 @@ func (f *Fs) itemToDirEntry(remote string, object *qs.KeyType, isDirectory bool)
 	return o, nil
 }
 
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+	if f.bucket != "" {
+		f.bucketOKMu.Lock()
+		f.bucketOK = true
+		f.bucketDeleted = false
+		f.bucketOKMu.Unlock()
+	}
+}
+
 // listDir lists files and directories to out
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	// List the objects and directories
@@ -535,6 +545,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return entries, nil
 }
 
@@ -606,6 +618,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return list.Flush()
 }
 

backend/s3/s3.go

@@ -590,6 +590,16 @@ func (f *Fs) itemToDirEntry(remote string, object *s3.Object, isDirectory bool)
 	return o, nil
 }
 
+// mark the bucket as being OK
+func (f *Fs) markBucketOK() {
+	if f.bucket != "" {
+		f.bucketOKMu.Lock()
+		f.bucketOK = true
+		f.bucketDeleted = false
+		f.bucketOKMu.Unlock()
+	}
+}
+
 // listDir lists files and directories to out
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	// List the objects and directories
@@ -606,6 +616,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return entries, nil
 }
 
@@ -673,6 +685,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	// bucket must be present if listing succeeded
+	f.markBucketOK()
 	return list.Flush()
 }
 

backend/swift/swift.go

@@ -407,6 +407,15 @@ func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error {
 	return err
 }
 
+// mark the container as being OK
+func (f *Fs) markContainerOK() {
+	if f.container != "" {
+		f.containerOKMu.Lock()
+		f.containerOK = true
+		f.containerOKMu.Unlock()
+	}
+}
+
 // listDir lists a single directory
 func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if f.container == "" {
@@ -420,6 +429,8 @@ func (f *Fs) listDir(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, err
 	}
+	// container must be present if listing succeeded
+	f.markContainerOK()
 	return entries, nil
 }
 
@@ -482,6 +493,8 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	// container must be present if listing succeeded
+	f.markContainerOK()
 	return list.Flush()
 }