mirror of
https://github.com/rclone/rclone.git
synced 2025-12-11 22:14:05 +01:00
dircache: simplify interface, fix corner cases and apply to backends
Dircache was changed to: - Remove special cases for the root directory - Remove Fatal errors - Call FindRoot on behalf of the user wherever possible - Bring up to modern Go standards Backends were changed to: - Remove calls to FindRoot - Change calls to FindRootAndPath to FindPath - Don't make special cases for the root This fixes several corner cases, for example removing a non existent directory if FindRoot hasn't been called.
This commit is contained in:
Nick Craig-Wood
@@ -1,4 +1,5 @@
|
||||
// Package dircache provides a simple cache for caching directory to path lookups
|
||||
// Package dircache provides a simple cache for caching directory ID
|
||||
// to path lookups and the inverse.
|
||||
package dircache
|
||||
|
||||
// _methods are called without the lock
|
||||
@@ -7,7 +8,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
@@ -17,19 +17,23 @@ import (
|
||||
|
||||
// DirCache caches paths to directory IDs and vice versa
|
||||
type DirCache struct {
|
||||
cacheMu sync.RWMutex
|
||||
cache map[string]string
|
||||
invCache map[string]string
|
||||
mu sync.Mutex
|
||||
fs DirCacher // Interface to find and make stuff
|
||||
trueRootID string // ID of the absolute root
|
||||
root string // the path we are working on
|
||||
rootID string // ID of the root directory
|
||||
rootParentID string // ID of the root's parent directory
|
||||
foundRoot bool // Whether we have found the root or not
|
||||
cacheMu sync.RWMutex // protects cache and invCache
|
||||
cache map[string]string
|
||||
invCache map[string]string
|
||||
|
||||
mu sync.Mutex // protects the below
|
||||
fs DirCacher // Interface to find and make directories
|
||||
trueRootID string // ID of the absolute root
|
||||
root string // the path the cache is rooted on
|
||||
rootID string // ID of the root directory
|
||||
rootParentID string // ID of the root's parent directory
|
||||
foundRoot bool // Whether we have found the root or not
|
||||
}
|
||||
|
||||
// DirCacher describes an interface for doing the low level directory work
|
||||
//
|
||||
// This should be implemented by the backend and will be called by the
|
||||
// dircache package when appropriate.
|
||||
type DirCacher interface {
|
||||
FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error)
|
||||
CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error)
|
||||
@@ -37,6 +41,15 @@ type DirCacher interface {
|
||||
|
||||
// New makes a DirCache
|
||||
//
|
||||
// This is created with the true root ID and the root path.
|
||||
//
|
||||
// In order to use the cache FindRoot() must be called on it without
|
||||
// error. This isn't done at initialization as it isn't known whether
|
||||
// the root and intermediate directories need to be created or not.
|
||||
//
|
||||
// Most of the utility functions wil call FindRoot() on the caller's
|
||||
// behalf with the create flag passed in.
|
||||
//
|
||||
// The cache is safe for concurrent use
|
||||
func New(root string, trueRootID string, fs DirCacher) *DirCache {
|
||||
d := &DirCache{
|
||||
@@ -74,23 +87,29 @@ func (dc *DirCache) String() string {
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// Get an ID given a path
|
||||
// Get a directory ID given a path
|
||||
//
|
||||
// Returns the ID and a boolean as to whether it was found or not in
|
||||
// the cache.
|
||||
func (dc *DirCache) Get(path string) (id string, ok bool) {
|
||||
dc.cacheMu.RLock()
|
||||
id, ok = dc.cache[path]
|
||||
dc.cacheMu.RUnlock()
|
||||
return
|
||||
return id, ok
|
||||
}
|
||||
|
||||
// GetInv gets a path given an ID
|
||||
// GetInv gets a path given a directory ID
|
||||
//
|
||||
// Returns the path and a boolean as to whether it was found or not in
|
||||
// the cache.
|
||||
func (dc *DirCache) GetInv(id string) (path string, ok bool) {
|
||||
dc.cacheMu.RLock()
|
||||
path, ok = dc.invCache[id]
|
||||
dc.cacheMu.RUnlock()
|
||||
return
|
||||
return path, ok
|
||||
}
|
||||
|
||||
// Put a path, id into the map
|
||||
// Put a (path, directory ID) pair into the cache
|
||||
func (dc *DirCache) Put(path, id string) {
|
||||
dc.cacheMu.Lock()
|
||||
dc.cache[path] = id
|
||||
@@ -98,7 +117,7 @@ func (dc *DirCache) Put(path, id string) {
|
||||
dc.cacheMu.Unlock()
|
||||
}
|
||||
|
||||
// Flush the map of all data
|
||||
// Flush the cache of all data
|
||||
func (dc *DirCache) Flush() {
|
||||
dc.cacheMu.Lock()
|
||||
dc.cache = make(map[string]string)
|
||||
@@ -106,9 +125,10 @@ func (dc *DirCache) Flush() {
|
||||
dc.cacheMu.Unlock()
|
||||
}
|
||||
|
||||
// FlushDir flushes the map of all data starting with dir
|
||||
// FlushDir flushes the map of all data starting with with the path
|
||||
// dir.
|
||||
//
|
||||
// If dir is empty then this is equivalent to calling ResetRoot
|
||||
// If dir is empty string then this is equivalent to calling ResetRoot
|
||||
func (dc *DirCache) FlushDir(dir string) {
|
||||
if dir == "" {
|
||||
dc.ResetRoot()
|
||||
@@ -159,39 +179,29 @@ func SplitPath(path string) (directory, leaf string) {
|
||||
//
|
||||
// If create is set it will make the directory if not found
|
||||
//
|
||||
// Algorithm:
|
||||
// Look in the cache for the path, if found return the pathID
|
||||
// If not found strip the last path off the path and recurse
|
||||
// Now have a parent directory id, so look in the parent for self and return it
|
||||
// It will call FindRoot if it hasn't been called already
|
||||
func (dc *DirCache) FindDir(ctx context.Context, path string, create bool) (pathID string, err error) {
|
||||
dc.mu.Lock()
|
||||
defer dc.mu.Unlock()
|
||||
err = dc._findRoot(ctx, create)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dc._findDir(ctx, path, create)
|
||||
}
|
||||
|
||||
// Look for the root and in the cache - safe to call without the mu
|
||||
func (dc *DirCache) _findDirInCache(path string) string {
|
||||
// fmt.Println("Finding",path,"create",create,"cache",cache)
|
||||
// Unlocked findDir
|
||||
//
|
||||
// Call with a lock on mu
|
||||
func (dc *DirCache) _findDir(ctx context.Context, path string, create bool) (pathID string, err error) {
|
||||
// If it is the root, then return it
|
||||
if path == "" {
|
||||
// fmt.Println("Root")
|
||||
return dc.rootID
|
||||
return dc.rootID, nil
|
||||
}
|
||||
|
||||
// If it is in the cache then return it
|
||||
pathID, ok := dc.Get(path)
|
||||
if ok {
|
||||
// fmt.Println("Cache hit on", path)
|
||||
return pathID
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// Unlocked findDir - must have mu
|
||||
func (dc *DirCache) _findDir(ctx context.Context, path string, create bool) (pathID string, err error) {
|
||||
pathID = dc._findDirInCache(path)
|
||||
if pathID != "" {
|
||||
return pathID, nil
|
||||
}
|
||||
|
||||
@@ -232,29 +242,48 @@ func (dc *DirCache) _findDir(ctx context.Context, path string, create bool) (pat
|
||||
|
||||
// FindPath finds the leaf and directoryID from a path
|
||||
//
|
||||
// Do not call FindPath with the root directory - it will return an error
|
||||
// If called with path == "" then it will return the ID of the parent
|
||||
// directory of the root and the leaf name of the root in that
|
||||
// directory. Note that it won't create the root directory in this
|
||||
// case even if create is true.
|
||||
//
|
||||
// If create is set parent directories will be created if they don't exist
|
||||
//
|
||||
// It will call FindRoot if it hasn't been called already
|
||||
func (dc *DirCache) FindPath(ctx context.Context, path string, create bool) (leaf, directoryID string, err error) {
|
||||
if path == "" {
|
||||
err = errors.New("internal error: can't call FindPath with root directory")
|
||||
return
|
||||
_, leaf = SplitPath(dc.root)
|
||||
directoryID, err = dc.RootParentID(ctx, create)
|
||||
} else {
|
||||
var directory string
|
||||
directory, leaf = SplitPath(path)
|
||||
directoryID, err = dc.FindDir(ctx, directory, create)
|
||||
}
|
||||
dc.mu.Lock()
|
||||
defer dc.mu.Unlock()
|
||||
directory, leaf := SplitPath(path)
|
||||
directoryID, err = dc._findDir(ctx, directory, create)
|
||||
return
|
||||
return leaf, directoryID, err
|
||||
}
|
||||
|
||||
// FindRoot finds the root directory if not already found
|
||||
//
|
||||
// If successful this changes the root of the cache from the true root
|
||||
// to the root specified by the path passed into New.
|
||||
//
|
||||
// Resets the root directory
|
||||
//
|
||||
// If create is set it will make the directory if not found
|
||||
func (dc *DirCache) FindRoot(ctx context.Context, create bool) error {
|
||||
dc.mu.Lock()
|
||||
defer dc.mu.Unlock()
|
||||
return dc._findRoot(ctx, create)
|
||||
}
|
||||
|
||||
// _findRoot finds the root directory if not already found
|
||||
//
|
||||
// Resets the root directory
|
||||
//
|
||||
// If create is set it will make the directory if not found
|
||||
//
|
||||
// Call with mu held
|
||||
func (dc *DirCache) _findRoot(ctx context.Context, create bool) error {
|
||||
if dc.foundRoot {
|
||||
return nil
|
||||
}
|
||||
@@ -277,51 +306,51 @@ func (dc *DirCache) FindRoot(ctx context.Context, create bool) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// FindRootAndPath finds the root first if not found then finds leaf and directoryID from a path
|
||||
//
|
||||
// If create is set parent directories will be created if they don't exist
|
||||
func (dc *DirCache) FindRootAndPath(ctx context.Context, path string, create bool) (leaf, directoryID string, err error) {
|
||||
err = dc.FindRoot(ctx, create)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return dc.FindPath(ctx, path, create)
|
||||
}
|
||||
|
||||
// FoundRoot returns whether the root directory has been found yet
|
||||
//
|
||||
// Call this from FindLeaf or CreateDir only
|
||||
func (dc *DirCache) FoundRoot() bool {
|
||||
dc.mu.Lock()
|
||||
defer dc.mu.Unlock()
|
||||
return dc.foundRoot
|
||||
}
|
||||
|
||||
// RootID returns the ID of the root directory
|
||||
//
|
||||
// This should be called after FindRoot
|
||||
func (dc *DirCache) RootID() string {
|
||||
// If create is set it will make the root directory if not found
|
||||
func (dc *DirCache) RootID(ctx context.Context, create bool) (ID string, err error) {
|
||||
dc.mu.Lock()
|
||||
defer dc.mu.Unlock()
|
||||
if !dc.foundRoot {
|
||||
log.Fatalf("Internal Error: RootID() called before FindRoot")
|
||||
err = dc._findRoot(ctx, create)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return dc.rootID
|
||||
return dc.rootID, nil
|
||||
}
|
||||
|
||||
// RootParentID returns the ID of the parent of the root directory
|
||||
//
|
||||
// This should be called after FindRoot
|
||||
func (dc *DirCache) RootParentID() (string, error) {
|
||||
// If create is set it will make the root parent directory if not found (but not the root)
|
||||
func (dc *DirCache) RootParentID(ctx context.Context, create bool) (ID string, err error) {
|
||||
dc.mu.Lock()
|
||||
defer dc.mu.Unlock()
|
||||
if !dc.foundRoot {
|
||||
return "", errors.New("internal error: RootID() called before FindRoot")
|
||||
if dc.root == "" {
|
||||
return "", errors.New("is root directory")
|
||||
}
|
||||
// Find the rootParentID without creating the root
|
||||
rootParent, _ := SplitPath(dc.root)
|
||||
rootParentID, err := dc._findDir(ctx, rootParent, create)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
dc.rootParentID = rootParentID
|
||||
} else {
|
||||
if dc.rootID == dc.trueRootID {
|
||||
return "", errors.New("is root directory")
|
||||
}
|
||||
}
|
||||
if dc.rootParentID == "" {
|
||||
return "", errors.New("internal error: didn't find rootParentID")
|
||||
}
|
||||
if dc.rootID == dc.trueRootID {
|
||||
return "", errors.New("is root directory")
|
||||
}
|
||||
return dc.rootParentID, nil
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user