vfs: implement --vfs-cache-max-size to limit the total size of the cache

vfs/cache_test.go

@@ -51,7 +51,7 @@ func itemAsString(c *cache) []string {
 	defer c.itemMu.Unlock()
 	var out []string
 	for name, item := range c.item {
-		out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d", name, item.isFile, item.opens))
+		out = append(out, fmt.Sprintf("name=%q isFile=%v opens=%d size=%d", name, item.isFile, item.opens, item.size))
 	}
 	sort.Strings(out)
 	return out
@@ -79,7 +79,7 @@ func TestCacheNew(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "potato", filepath.Base(p))
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
+		`name="" isFile=false opens=0 size=0`,
 	}, itemAsString(c))
 
 	fi, err := os.Stat(filepath.Dir(p))
@@ -95,26 +95,26 @@ func TestCacheNew(t *testing.T) {
 	// updateTime
 	//.. before
 	t1 := time.Now().Add(-60 * time.Minute)
-	c.updateTime("potato", t1)
+	c.updateStat("potato", t1, 0)
 	item = c.get("potato")
 	assert.NotEqual(t, t1, item.atime)
 	assert.Equal(t, 0, item.opens)
 	//..after
 	t2 := time.Now().Add(60 * time.Minute)
-	c.updateTime("potato", t2)
+	c.updateStat("potato", t2, 0)
 	item = c.get("potato")
 	assert.Equal(t, t2, item.atime)
 	assert.Equal(t, 0, item.opens)
 
 	// open
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 	c.open("/potato")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
 	item = c.get("potato")
 	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
@@ -132,11 +132,11 @@ func TestCacheNew(t *testing.T) {
 	// updateAtimes
 	item = c.get("potato")
 	item.atime = time.Now().Add(-24 * time.Hour)
-	err = c.updateAtimes()
+	err = c.updateStats()
 	require.NoError(t, err)
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=5`,
 	}, itemAsString(c))
 	item = c.get("potato")
 	assert.Equal(t, atime, item.atime)
@@ -146,11 +146,11 @@ func TestCacheNew(t *testing.T) {
 	c.itemMu.Lock()
 	delete(c.item, "potato") // remove from cache
 	c.itemMu.Unlock()
-	err = c.updateAtimes()
+	err = c.updateStats()
 	require.NoError(t, err)
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=0`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=0 size=5`,
 	}, itemAsString(c))
 	item = c.get("potato")
 	assert.Equal(t, atime, item.atime)
@@ -165,18 +165,18 @@ func TestCacheNew(t *testing.T) {
 
 	// close
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=5`,
 	}, itemAsString(c))
-	c.updateTime("potato", t2)
+	c.updateStat("potato", t2, 6)
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=6`,
 	}, itemAsString(c))
 	c.close("potato/")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="potato" isFile=true opens=0 size=5`,
 	}, itemAsString(c))
 	item = c.get("potato")
 	assert.WithinDuration(t, time.Now(), item.atime, time.Second)
@@ -216,23 +216,23 @@ func TestCacheOpens(t *testing.T) {
 	assert.Equal(t, []string(nil), itemAsString(c))
 	c.open("potato")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
 	c.open("potato")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=2`,
-		`name="potato" isFile=true opens=2`,
+		`name="" isFile=false opens=2 size=0`,
+		`name="potato" isFile=true opens=2 size=0`,
 	}, itemAsString(c))
 	c.close("potato")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
 	c.close("potato")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 
 	c.open("potato")
@@ -240,34 +240,34 @@ func TestCacheOpens(t *testing.T) {
 	c.open("a/b/c/d/e/two")
 	c.open("a/b/c/d/e/f/three")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=4`,
-		`name="a" isFile=false opens=3`,
-		`name="a/b" isFile=false opens=3`,
-		`name="a/b/c" isFile=false opens=3`,
-		`name="a/b/c/d" isFile=false opens=3`,
-		`name="a/b/c/d/e" isFile=false opens=2`,
-		`name="a/b/c/d/e/f" isFile=false opens=1`,
-		`name="a/b/c/d/e/f/three" isFile=true opens=1`,
-		`name="a/b/c/d/e/two" isFile=true opens=1`,
-		`name="a/b/c/d/one" isFile=true opens=1`,
-		`name="potato" isFile=true opens=1`,
+		`name="" isFile=false opens=4 size=0`,
+		`name="a" isFile=false opens=3 size=0`,
+		`name="a/b" isFile=false opens=3 size=0`,
+		`name="a/b/c" isFile=false opens=3 size=0`,
+		`name="a/b/c/d" isFile=false opens=3 size=0`,
+		`name="a/b/c/d/e" isFile=false opens=2 size=0`,
+		`name="a/b/c/d/e/f" isFile=false opens=1 size=0`,
+		`name="a/b/c/d/e/f/three" isFile=true opens=1 size=0`,
+		`name="a/b/c/d/e/two" isFile=true opens=1 size=0`,
+		`name="a/b/c/d/one" isFile=true opens=1 size=0`,
+		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
 	c.close("potato")
 	c.close("a/b/c/d/one")
 	c.close("a/b/c/d/e/two")
 	c.close("a/b/c//d/e/f/three")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="a" isFile=false opens=0`,
-		`name="a/b" isFile=false opens=0`,
-		`name="a/b/c" isFile=false opens=0`,
-		`name="a/b/c/d" isFile=false opens=0`,
-		`name="a/b/c/d/e" isFile=false opens=0`,
-		`name="a/b/c/d/e/f" isFile=false opens=0`,
-		`name="a/b/c/d/e/f/three" isFile=true opens=0`,
-		`name="a/b/c/d/e/two" isFile=true opens=0`,
-		`name="a/b/c/d/one" isFile=true opens=0`,
-		`name="potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="a" isFile=false opens=0 size=0`,
+		`name="a/b" isFile=false opens=0 size=0`,
+		`name="a/b/c" isFile=false opens=0 size=0`,
+		`name="a/b/c/d" isFile=false opens=0 size=0`,
+		`name="a/b/c/d/e" isFile=false opens=0 size=0`,
+		`name="a/b/c/d/e/f" isFile=false opens=0 size=0`,
+		`name="a/b/c/d/e/f/three" isFile=true opens=0 size=0`,
+		`name="a/b/c/d/e/two" isFile=true opens=0 size=0`,
+		`name="a/b/c/d/one" isFile=true opens=0 size=0`,
+		`name="potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 }
 
@@ -289,9 +289,9 @@ func TestCacheOpenMkdir(t *testing.T) {
 	c.open("sub/potato")
 
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="sub" isFile=false opens=1`,
-		`name="sub/potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="sub" isFile=false opens=1 size=0`,
+		`name="sub/potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
 
 	// mkdir
@@ -299,9 +299,9 @@ func TestCacheOpenMkdir(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "potato", filepath.Base(p))
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=1`,
-		`name="sub" isFile=false opens=1`,
-		`name="sub/potato" isFile=true opens=1`,
+		`name="" isFile=false opens=1 size=0`,
+		`name="sub" isFile=false opens=1 size=0`,
+		`name="sub/potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
 
 	// test directory exists
@@ -321,13 +321,14 @@ func TestCacheOpenMkdir(t *testing.T) {
 	c.close("sub/potato")
 
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="sub" isFile=false opens=0`,
-		`name="sub/potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 
 	// clean the cache
 	c.purgeOld(-10 * time.Second)
+	c.purgeEmptyDirs()
 
 	assert.Equal(t, []string(nil), itemAsString(c))
 
@@ -350,24 +351,24 @@ func TestCacheCacheDir(t *testing.T) {
 
 	c.cacheDir("dir")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="dir" isFile=false opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="dir" isFile=false opens=0 size=0`,
 	}, itemAsString(c))
 
 	c.cacheDir("dir/sub")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="dir" isFile=false opens=0`,
-		`name="dir/sub" isFile=false opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="dir" isFile=false opens=0 size=0`,
+		`name="dir/sub" isFile=false opens=0 size=0`,
 	}, itemAsString(c))
 
 	c.cacheDir("dir/sub2/subsub2")
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="dir" isFile=false opens=0`,
-		`name="dir/sub" isFile=false opens=0`,
-		`name="dir/sub2" isFile=false opens=0`,
-		`name="dir/sub2/subsub2" isFile=false opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="dir" isFile=false opens=0 size=0`,
+		`name="dir/sub" isFile=false opens=0 size=0`,
+		`name="dir/sub2" isFile=false opens=0 size=0`,
+		`name="dir/sub2/subsub2" isFile=false opens=0 size=0`,
 	}, itemAsString(c))
 }
 
@@ -395,7 +396,8 @@ func TestCachePurgeOld(t *testing.T) {
 	}
 
 	removed = nil
-	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	c._purgeOld(-10*time.Second, removeFile)
+	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string(nil), removed)
 
 	c.open("sub/dir2/potato2")
@@ -404,17 +406,18 @@ func TestCachePurgeOld(t *testing.T) {
 	c.open("sub/dir/potato")
 
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=2`,
-		`name="sub" isFile=false opens=2`,
-		`name="sub/dir" isFile=false opens=2`,
-		`name="sub/dir/potato" isFile=true opens=2`,
-		`name="sub/dir2" isFile=false opens=0`,
-		`name="sub/dir2/potato2" isFile=true opens=0`,
+		`name="" isFile=false opens=2 size=0`,
+		`name="sub" isFile=false opens=2 size=0`,
+		`name="sub/dir" isFile=false opens=2 size=0`,
+		`name="sub/dir/potato" isFile=true opens=2 size=0`,
+		`name="sub/dir2" isFile=false opens=0 size=0`,
+		`name="sub/dir2/potato2" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 
 	removed = nil
 	removedDir = true
-	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	c._purgeOld(-10*time.Second, removeFile)
+	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string{
 		"sub/dir2/potato2",
 		"sub/dir2/",
@@ -424,33 +427,36 @@ func TestCachePurgeOld(t *testing.T) {
 
 	removed = nil
 	removedDir = true
-	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	c._purgeOld(-10*time.Second, removeFile)
+	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string(nil), removed)
 
 	c.close("sub/dir/potato")
 
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="sub" isFile=false opens=0`,
-		`name="sub/dir" isFile=false opens=0`,
-		`name="sub/dir/potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/dir" isFile=false opens=0 size=0`,
+		`name="sub/dir/potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 
 	removed = nil
 	removedDir = false
-	c._purgeOld(10*time.Second, removeFile, removeDir)
+	c._purgeOld(10*time.Second, removeFile)
+	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string(nil), removed)
 
 	assert.Equal(t, []string{
-		`name="" isFile=false opens=0`,
-		`name="sub" isFile=false opens=0`,
-		`name="sub/dir" isFile=false opens=0`,
-		`name="sub/dir/potato" isFile=true opens=0`,
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/dir" isFile=false opens=0 size=0`,
+		`name="sub/dir/potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 
 	removed = nil
 	removedDir = true
-	c._purgeOld(-10*time.Second, removeFile, removeDir)
+	c._purgeOld(-10*time.Second, removeFile)
+	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string{
 		"sub/dir/potato",
 		"sub/dir/",
@@ -460,3 +466,157 @@ func TestCachePurgeOld(t *testing.T) {
 
 	assert.Equal(t, []string(nil), itemAsString(c))
 }
+
+func TestCachePurgeOverQuota(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Disable the cache cleaner as it interferes with these tests
+	opt := DefaultOpt
+	opt.CachePollInterval = 0
+	c, err := newCache(ctx, r.Fremote, &opt)
+	require.NoError(t, err)
+
+	// Test funcs
+	var removed []string
+	remove := func(name string) {
+		removed = append(removed, name)
+		c.remove(name)
+	}
+
+	removed = nil
+	c._purgeOverQuota(-1, remove)
+	assert.Equal(t, []string(nil), removed)
+
+	removed = nil
+	c._purgeOverQuota(0, remove)
+	assert.Equal(t, []string(nil), removed)
+
+	removed = nil
+	c._purgeOverQuota(1, remove)
+	assert.Equal(t, []string(nil), removed)
+
+	// Make some test files
+	c.open("sub/dir/potato")
+	p, err := c.mkdir("sub/dir/potato")
+	require.NoError(t, err)
+	err = ioutil.WriteFile(p, []byte("hello"), 0600)
+	require.NoError(t, err)
+
+	p, err = c.mkdir("sub/dir2/potato2")
+	c.open("sub/dir2/potato2")
+	require.NoError(t, err)
+	err = ioutil.WriteFile(p, []byte("hello2"), 0600)
+	require.NoError(t, err)
+	// make it definitely after
+	t1 := time.Now().Add(10 * time.Second)
+	c.updateStat("sub/dir2/potato2", t1, 0)
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=2 size=0`,
+		`name="sub" isFile=false opens=2 size=0`,
+		`name="sub/dir" isFile=false opens=1 size=0`,
+		`name="sub/dir/potato" isFile=true opens=1 size=0`,
+		`name="sub/dir2" isFile=false opens=1 size=0`,
+		`name="sub/dir2/potato2" isFile=true opens=1 size=0`,
+	}, itemAsString(c))
+
+	// Check nothing removed
+	removed = nil
+	c._purgeOverQuota(1, remove)
+	assert.Equal(t, []string(nil), removed)
+
+	// Close the files
+	c.close("sub/dir/potato")
+	c.close("sub/dir2/potato2")
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/dir" isFile=false opens=0 size=0`,
+		`name="sub/dir/potato" isFile=true opens=0 size=5`,
+		`name="sub/dir2" isFile=false opens=0 size=0`,
+		`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
+	}, itemAsString(c))
+
+	// Update the stats to read the total size
+	err = c.updateStats()
+	require.NoError(t, err)
+	assert.Equal(t, int64(11), c.used)
+
+	// Check only potato removed to get below quota
+	removed = nil
+	c._purgeOverQuota(10, remove)
+	assert.Equal(t, []string{
+		"sub/dir/potato",
+	}, removed)
+	assert.Equal(t, int64(6), c.used)
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/dir" isFile=false opens=0 size=0`,
+		`name="sub/dir2" isFile=false opens=0 size=0`,
+		`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
+	}, itemAsString(c))
+
+	// Put potato back
+	c.open("sub/dir/potato")
+	p, err = c.mkdir("sub/dir/potato")
+	require.NoError(t, err)
+	err = ioutil.WriteFile(p, []byte("hello"), 0600)
+	require.NoError(t, err)
+	c.close("sub/dir/potato")
+	// make it definitely after
+	t2 := t1.Add(20 * time.Second)
+	c.updateStat("sub/dir/potato", t2, 5)
+
+	// Update the stats to read the total size
+	err = c.updateStats()
+	require.NoError(t, err)
+	assert.Equal(t, int64(11), c.used)
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/dir" isFile=false opens=0 size=0`,
+		`name="sub/dir/potato" isFile=true opens=0 size=5`,
+		`name="sub/dir2" isFile=false opens=0 size=0`,
+		`name="sub/dir2/potato2" isFile=true opens=0 size=6`,
+	}, itemAsString(c))
+
+	// Check only potato2 removed to get below quota
+	removed = nil
+	c._purgeOverQuota(10, remove)
+	assert.Equal(t, []string{
+		"sub/dir2/potato2",
+	}, removed)
+	assert.Equal(t, int64(5), c.used)
+	c.purgeEmptyDirs()
+
+	assert.Equal(t, []string{
+		`name="" isFile=false opens=0 size=0`,
+		`name="sub" isFile=false opens=0 size=0`,
+		`name="sub/dir" isFile=false opens=0 size=0`,
+		`name="sub/dir/potato" isFile=true opens=0 size=5`,
+	}, itemAsString(c))
+
+	// Now purge everything
+	removed = nil
+	c._purgeOverQuota(1, remove)
+	assert.Equal(t, []string{
+		"sub/dir/potato",
+	}, removed)
+	assert.Equal(t, int64(0), c.used)
+	c.purgeEmptyDirs()
+
+	assert.Equal(t, []string(nil), itemAsString(c))
+
+	// Check nothing left behind
+	c.clean()
+	assert.Equal(t, int64(0), c.used)
+	assert.Equal(t, []string(nil), itemAsString(c))
+}