package azurefiles import ( "context" "errors" "fmt" "io" "log" "path" "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/directory" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/file" "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/fileerror" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/hash" ) const sleepDurationBetweenRecursiveMkdirPutCalls = time.Millisecond * 500 const fourTbInBytes = 4398046511104 // NewObject finds the Object at remote. If it can't be found // it returns the error fs.ErrorObjectNotFound. // // Does not return ErrorIsDir when a directory exists instead of file. since the documentation // for [rclone.fs.Fs.NewObject] rqeuires no extra work to determine whether it is directory // // Inspired by azureblob store, this initiates a network request and returns an error if object is not found. func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { fileClient := f.fileClient(remote) resp, err := fileClient.GetProperties(ctx, nil) if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) { return nil, fs.ErrorObjectNotFound } else if err != nil { return nil, fmt.Errorf("unable to find object remote=%s : %w", remote, err) } ob := objectInstance(f, remote, *resp.ContentLength, resp.ContentMD5, *resp.FileLastWriteTime) return &ob, nil } // Mkdir creates nested directories as indicated by test FsMkdirRmdirSubdir // TODO: write custom test case where parent directories are created // Mkdir creates the container if it doesn't exist func (f *Fs) Mkdir(ctx context.Context, remote string) error { return f.mkdirRelativeToRootOfShare(ctx, f.decodedFullPath(remote)) } // rclone completes commands such as rclone copy localdir remote:parentcontainer/childcontainer // where localdir is a tree of files and directories. The above command is expected to complete even // when parentcontainer and childcontainer directors do not exist on the remote. The following // code with emphasis on fullPathRelativeToShareRoot is written to handle such cases by recursiely creating // parent directories all the way to the root of the share // // When path argument is an empty string, windows and linux return and error. However, this // implementation does not return an error func (f *Fs) mkdirRelativeToRootOfShare(ctx context.Context, fullPathRelativeToShareRoot string) error { fp := fullPathRelativeToShareRoot if fp == "" { return nil } dirClient := f.newSubdirectoryClientFromEncodedPathRelativeToShareRoot(f.encodePath(fp)) // now := time.Now() // smbProps := &file.SMBProperties{ // LastWriteTime: &now, // } // dirCreateOptions := &directory.CreateOptions{ // FileSMBProperties: smbProps, // } _, createDirErr := dirClient.Create(ctx, nil) if fileerror.HasCode(createDirErr, fileerror.ParentNotFound) { parentDir := path.Dir(fp) if parentDir == fp { log.Fatal("This will lead to infinite recursion since parent and remote are equal") } makeParentErr := f.mkdirRelativeToRootOfShare(ctx, parentDir) if makeParentErr != nil { return fmt.Errorf("could not make parent of %s : %w", fp, makeParentErr) } log.Printf("Mkdir: waiting for %s after making parent=%s", sleepDurationBetweenRecursiveMkdirPutCalls.String(), parentDir) time.Sleep(sleepDurationBetweenRecursiveMkdirPutCalls) return f.mkdirRelativeToRootOfShare(ctx, fp) } else if fileerror.HasCode(createDirErr, fileerror.ResourceAlreadyExists) { return nil } else if createDirErr != nil { return fmt.Errorf("unable to MkDir: %w", createDirErr) } return nil } // Rmdir deletes the root folder // // Returns an error if it isn't empty func (f *Fs) Rmdir(ctx context.Context, remote string) error { dirClient := f.dirClient(remote) _, err := dirClient.Delete(ctx, nil) if err != nil { if fileerror.HasCode(err, fileerror.DirectoryNotEmpty) { return fs.ErrorDirectoryNotEmpty } else if fileerror.HasCode(err, fileerror.ResourceNotFound) { return fs.ErrorDirNotFound } return fmt.Errorf("could not rmdir dir=\"%s\" : %w", remote, err) } return nil } // Put the object // // Copies the reader in to the new object. This new object is returned. // // The new object may have been created if an error is returned // TODO: when file.CLient.Creat is being used, provide HTTP headesr such as content type and content MD5 // TODO: maybe replace PUT with NewObject + Update // TODO: in case file is created but there is a problem on upload, what happens // TODO: what happens when file already exists at the location func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { if src.Size() > fourTbInBytes { return nil, fmt.Errorf("max supported file size is 4TB. provided size is %d", src.Size()) } else if src.Size() < 0 { // TODO: what should happened when src.Size == 0 return nil, fmt.Errorf("src.Size is a required to be a whole number : %d", src.Size()) } fc := f.fileClient(src.Remote()) _, createErr := fc.Create(ctx, src.Size(), nil) if fileerror.HasCode(createErr, fileerror.ParentNotFound) { parentDir := path.Dir(src.Remote()) if mkDirErr := f.Mkdir(ctx, parentDir); mkDirErr != nil { return nil, fmt.Errorf("unable to make parent directories : %w", mkDirErr) } log.Printf("Mkdir: waiting for %s after making parent=%s", sleepDurationBetweenRecursiveMkdirPutCalls.String(), parentDir) time.Sleep(sleepDurationBetweenRecursiveMkdirPutCalls) return f.Put(ctx, in, src, options...) } else if createErr != nil { return nil, fmt.Errorf("unable to create file : %w", createErr) } obj := &Object{ common: common{ f: f, remote: src.Remote(), }, } if updateErr := obj.upload(ctx, in, src, true, options...); updateErr != nil { err := fmt.Errorf("while executing update after creating file as part of fs.Put : %w", updateErr) if _, delErr := fc.Delete(ctx, nil); delErr != nil { return nil, errors.Join(delErr, updateErr) } return obj, err } return obj, nil } // Name of the remote (as passed into NewFs) func (f *Fs) Name() string { return f.name } // Root of the remote (as passed into NewFs) func (f *Fs) Root() string { return f.root } // String converts this Fs to a string func (f *Fs) String() string { return fmt.Sprintf("azurefiles root '%s'", f.root) } // Precision return the precision of this Fs // // One second. FileREST API times are in RFC1123 which in the example shows a precision of seconds // Source: https://learn.microsoft.com/en-us/rest/api/storageservices/representation-of-date-time-values-in-headers func (f *Fs) Precision() time.Duration { return time.Second } // Hashes returns the supported hash sets. // // MD5: since it is listed as header in the response for file properties // Source: https://learn.microsoft.com/en-us/rest/api/storageservices/get-file-properties func (f *Fs) Hashes() hash.Set { return hash.NewHashSet(hash.MD5) } // Features returns the optional features of this Fs // // TODO: add features:- public link, SlowModTime, SlowHash, // ReadMetadata, WriteMetadata,UserMetadata,PutUnchecked, PutStream // PartialUploads: Maybe???? // FileID and DirectoryID can be implemented. They are atleast returned as part of listing response func (f *Fs) Features() *fs.Features { return &fs.Features{ CanHaveEmptyDirectories: true, // Copy: func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { // return f.CopyFile(ctx, src, remote) // }, } } // List the objects and directories in dir into entries. The entries can be // returned in any order but should be for a complete directory. // // dir should be "" to list the root, and should not have trailing slashes. // // This should return ErrDirNotFound if the directory isn't found. // // TODO: handle case regariding "" and "/". I remember reading about them somewhere func (f *Fs) List(ctx context.Context, remote string) (fs.DirEntries, error) { var entries fs.DirEntries subDirClient := f.dirClient(remote) // Checking whether directory exists _, err := subDirClient.GetProperties(ctx, nil) if fileerror.HasCode(err, fileerror.ParentNotFound, fileerror.ResourceNotFound) { return entries, fs.ErrorDirNotFound } else if err != nil { return entries, err } pager := subDirClient.NewListFilesAndDirectoriesPager(listFilesAndDirectoriesOptions) for pager.More() { resp, err := pager.NextPage(ctx) if err != nil { return entries, err } for _, dir := range resp.Segment.Directories { de := &Directory{ common{f: f, remote: path.Join(remote, f.decodePath(*dir.Name)), properties: properties{ lastWriteTime: *dir.Properties.LastWriteTime, }}, } entries = append(entries, de) } for _, file := range resp.Segment.Files { de := &Object{ common{f: f, remote: path.Join(remote, f.decodePath(*file.Name)), properties: properties{ contentLength: *file.Properties.ContentLength, lastWriteTime: *file.Properties.LastWriteTime, }}, } entries = append(entries, de) } } return entries, nil } type encodedPath string func (f *Fs) decodedFullPath(decodedRemote string) string { return path.Join(f.root, decodedRemote) } func (f *Fs) dirClient(decodedRemote string) *directory.Client { fullPathDecoded := f.decodedFullPath(decodedRemote) fullPathEncoded := f.encodePath(fullPathDecoded) return f.newSubdirectoryClientFromEncodedPathRelativeToShareRoot(fullPathEncoded) } func (f *Fs) newSubdirectoryClientFromEncodedPathRelativeToShareRoot(p encodedPath) *directory.Client { return f.shareRootDirClient.NewSubdirectoryClient(string(p)) } func (f *Fs) fileClient(decodedRemote string) *file.Client { fullPathDecoded := f.decodedFullPath(decodedRemote) fullPathEncoded := f.encodePath(fullPathDecoded) return f.fileClientFromEncodedPathRelativeToShareRoot(fullPathEncoded) } func (f *Fs) fileClientFromEncodedPathRelativeToShareRoot(p encodedPath) *file.Client { return f.shareRootDirClient.NewFileClient(string(p)) } func (f *Fs) encodePath(p string) encodedPath { return encodedPath(f.opt.Enc.FromStandardPath(p)) } func (f *Fs) decodePath(p string) string { return f.opt.Enc.ToStandardPath(p) } // on 20231019 at 1324 work to be continued at trying to fix FAIL: TestIntegration/FsMkdir/FsPutFiles/FromRoot