pagecache: add test suite

Author: sahib

catfs/mio/pagecache/cache.go

@@ -4,9 +4,21 @@ import (
 	"github.com/sahib/brig/catfs/mio/pagecache/page"
 )
 
+// Cache is the backing layer that stores pages in memory
+// or whatever medium it choses to use.
 type Cache interface {
-	Evict(inode int32) error
-	Lookup(inode, page int32) (*page.Page, error)
-	Merge(inode, pageID, off int32, buf []byte) error
+	// Lookup returns a cached page, identified by `inode` and `page`.
+	// If there is no such page page.ErrCacheMiss is returned.
+	Lookup(inode int64, page uint32) (*page.Page, error)
+
+	// Merge the existing cache contents with the new write
+	// to `pageID`, starting at `pageOff` and with the contents of `buf`.
+	Merge(inode int64, pageID, pageOff uint32, buf []byte) error
+
+	// Evict clears cached pages for `inode`. `size` can be used
+	// to clear only up to a certain size.
+	Evict(inode, size int64) error
+
+	// Close the cache and free up all resources.
 	Close() error
 }

catfs/mio/pagecache/dircache/l1.go renamed to catfs/mio/pagecache/mdcache/l1.go

@@ -1,8 +1,7 @@
-package dircache
+package mdcache
 
 import (
 	"container/list"
-	"errors"
 	"fmt"
 
 	"github.com/sahib/brig/catfs/mio/pagecache/page"
@@ -37,31 +36,41 @@ type l1item struct {
 type l1cache struct {
 	m         map[pageKey]l1item
 	k         *list.List
-	l2        *l2cache
+	l2        cacheLayer
 	maxMemory int64
 }
 
-func newL1Cache(l2 *l2cache, maxMemory int64) (*l1cache, error) {
+func newL1Cache(l2 cacheLayer, maxMemory int64) (*l1cache, error) {
 	return &l1cache{
 		maxMemory: maxMemory,
 		l2:        l2,
 		k:         list.New(),
+		m:         make(map[pageKey]l1item),
 	}, nil
 }
 
 func (c *l1cache) Set(pk pageKey, p *page.Page) error {
-	c.m[pk] = l1item{
-		Page: p,
-		Link: c.k.PushBack(pk),
+	existingItem, ok := c.m[pk]
+	if !ok {
+		// new content:
+		c.m[pk] = l1item{
+			Page: p,
+			Link: c.k.PushBack(pk),
+		}
+	} else {
+		// do not push another page key,
+		// c.k needs to have unique keys only.
+		c.m[pk] = l1item{
+			Page: p,
+			Link: existingItem.Link,
+		}
+
+		// prioritize this one more.
+		c.k.MoveToBack(existingItem.Link)
 	}
 
 	maxPages := c.maxMemory / (page.Size + page.Meta)
 	if int64(len(c.m)) > maxPages {
-		if c.l2 == nil {
-			// just in case l2 cache was not given:
-			return errors.New("cache is full")
-		}
-
 		oldPkIface := c.k.Remove(c.k.Front())
 		oldPk, ok := oldPkIface.(pageKey)
 		if !ok {
@@ -73,7 +82,13 @@ func (c *l1cache) Set(pk pageKey, p *page.Page) error {
 		if !ok {
 			// c.m and c.k got out of sync.
 			// this is very likely a bug.
-			return fmt.Errorf("l1: key in key list, but not in map")
+			return fmt.Errorf("l1: key in key list, but not in map: %v", oldPk)
+		}
+
+		if c.l2 == nil {
+			// nil-interface for l2: loose pages in that case.
+			// that may be valid if no disk can be used.
+			return nil
 		}
 
 		// move old page to more persistent cache layer:
@@ -107,5 +122,8 @@ func (c *l1cache) Del(pks []pageKey) error {
 }
 
 func (c *l1cache) Close() error {
+	// help GC if caller somehow still retains a reference:
+	c.m = nil
+	c.k = nil
 	return nil
 }

catfs/mio/pagecache/mdcache/l1_test.go

@@ -0,0 +1,76 @@
+package mdcache
+
+import (
+	"testing"
+
+	"github.com/sahib/brig/catfs/mio/pagecache/page"
+	"github.com/stretchr/testify/require"
+)
+
+func withL1Cache(t *testing.T, fn func(l1, backing *l1cache)) {
+	// some fake in-mem cache that stores everything that got removed
+	// out of l1 due to size restrictions.
+	backing, err := newL1Cache(nil, int64(^uint64(0)>>1))
+	require.NoError(t, err)
+
+	l1, err := newL1Cache(backing, 4*(page.Size+page.Meta))
+	require.NoError(t, err)
+
+	fn(l1, backing)
+
+	require.NoError(t, l1.Close())
+}
+
+func TestL1GetSetDel(t *testing.T) {
+	// NOTE: Only covers the very basic usage.
+	withL1Cache(t, func(l1, _ *l1cache) {
+		pk := pageKey{1, 0}
+		_, err := l1.Get(pk)
+		require.Error(t, page.ErrCacheMiss)
+
+		pset := dummyPage(0, 1024)
+		require.NoError(t, l1.Set(pk, pset))
+
+		pgot, err := l1.Get(pk)
+		require.NoError(t, err)
+
+		require.Equal(t, pset.Data, pgot.Data)
+		require.Equal(t, pset.Extents, pgot.Extents)
+
+		require.NoError(t, l1.Del([]pageKey{pk}))
+		_, err = l1.Get(pk)
+		require.Error(t, page.ErrCacheMiss)
+	})
+}
+
+func TestL1SwapPriority(t *testing.T) {
+	withL1Cache(t, func(l1, backing *l1cache) {
+		// Insert 8 pages, only 4 can stay in l1.
+		for idx := 0; idx < 8; idx++ {
+			pk := pageKey{1, uint32(idx)}
+			require.NoError(t, l1.Set(pk, dummyPage(0, uint32((idx+1)*100))))
+		}
+
+		for idx := 0; idx < 4; idx++ {
+			pk := pageKey{1, uint32(idx)}
+			_, err := l1.Get(pk)
+			require.Error(t, err, page.ErrCacheMiss)
+
+			// should be in backing store, check:
+			p, err := backing.Get(pk)
+			require.NoError(t, err)
+
+			expected := dummyPage(0, uint32((idx+1)*100))
+			require.Equal(t, expected, p)
+		}
+
+		for idx := 4; idx < 8; idx++ {
+			pk := pageKey{1, uint32(idx)}
+			p, err := l1.Get(pk)
+			require.NoError(t, err)
+
+			expected := dummyPage(0, uint32((idx+1)*100))
+			require.Equal(t, expected, p)
+		}
+	})
+}

catfs/mio/pagecache/dircache/l2.go renamed to catfs/mio/pagecache/mdcache/l2.go

@@ -1,4 +1,4 @@
-package dircache
+package mdcache
 
 import (
 	"fmt"
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/sahib/brig/catfs/mio/pagecache/page"
+	log "github.com/sirupsen/logrus"
 )
 
 type l2cache struct {
@@ -35,16 +36,12 @@ func newL2Cache(dir string) (*l2cache, error) {
 }
 
 func (c *l2cache) Set(pk pageKey, p *page.Page) error {
-	return c.SetData(pk.String(), p.AsBytes())
-}
-
-func (c *l2cache) SetData(key string, pdata []byte) error {
 	if c == nil {
 		return nil
 	}
 
-	path := filepath.Join(c.dir, key)
-	return ioutil.WriteFile(path, pdata, 0600)
+	path := filepath.Join(c.dir, pk.String())
+	return ioutil.WriteFile(path, p.AsBytes(), 0600)
 }
 
 func (c *l2cache) Get(pk pageKey) (*page.Page, error) {
@@ -70,6 +67,7 @@ func (c *l2cache) Del(pks []pageKey) error {
 		path := filepath.Join(c.dir, pk.String())
 		if err := os.Remove(path); err != nil {
 			// only log, we want to get rid of more old data.
+			log.Warnf("page l2: failed to delete %s", path)
 		}
 	}
 

catfs/mio/pagecache/mdcache/l2_test.go

@@ -0,0 +1,65 @@
+package mdcache
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/sahib/brig/catfs/mio/pagecache/page"
+	"github.com/sahib/brig/util/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+func dummyPage(off, length uint32) *page.Page {
+	buf := testutil.CreateDummyBuf(int64(length))
+	return page.New(off, buf)
+}
+
+func withL2Cache(t *testing.T, fn func(l2 *l2cache)) {
+	tmpDir, err := ioutil.TempDir("", "brig-page-l2")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	l2, err := newL2Cache(tmpDir)
+	require.NoError(t, err)
+
+	fn(l2)
+
+	// double check we do not waste any storage:
+	require.NoError(t, l2.Close())
+	_, err = os.Stat(tmpDir)
+	require.True(t, os.IsNotExist(err))
+}
+
+func TestL2GetSetDel(t *testing.T) {
+	withL2Cache(t, func(l2 *l2cache) {
+		pk := pageKey{1, 0}
+		_, err := l2.Get(pk)
+		require.Error(t, page.ErrCacheMiss)
+
+		pset := dummyPage(0, 1024)
+		require.NoError(t, l2.Set(pk, pset))
+
+		pgot, err := l2.Get(pk)
+		require.NoError(t, err)
+
+		require.Equal(t, pset.Data, pgot.Data)
+		require.Equal(t, pset.Extents, pgot.Extents)
+
+		require.NoError(t, l2.Del([]pageKey{pk}))
+		_, err = l2.Get(pk)
+		require.Error(t, page.ErrCacheMiss)
+	})
+}
+
+func TestL2Nil(t *testing.T) {
+	// l2 is optional, so a nil l2 cache should "work":
+	l2, err := newL2Cache("")
+	require.NoError(t, err)
+
+	_, err = l2.Get(pageKey{0, 1})
+	require.Error(t, page.ErrCacheMiss)
+
+	require.NoError(t, l2.Set(pageKey{0, 1}, dummyPage(0, 1024)))
+	require.NoError(t, l2.Del([]pageKey{{0, 1}}))
+}