introduce a freelist interface

This introduces an interface for the freelist, splits it into two concrete
implementations.

fixes #773

Signed-off-by: Thomas Jungblut <[email protected]>

Author: tjungblu

allocate_test.go

@@ -4,12 +4,13 @@ import (
 	"testing"
 
 	"go.etcd.io/bbolt/internal/common"
+	"go.etcd.io/bbolt/internal/freelist"
 )
 
 func TestTx_allocatePageStats(t *testing.T) {
-	f := newTestFreelist()
+	f := freelist.NewFreelist(freelist.FreelistArrayType)
 	ids := []common.Pgid{2, 3}
-	f.readIDs(ids)
+	f.Init(ids)
 
 	tx := &Tx{
 		db: &DB{
@@ -22,7 +23,7 @@ func TestTx_allocatePageStats(t *testing.T) {
 
 	txStats := tx.Stats()
 	prePageCnt := txStats.GetPageCount()
-	allocateCnt := f.free_count()
+	allocateCnt := f.Count()
 
 	if _, err := tx.allocate(allocateCnt); err != nil {
 		t.Fatal(err)

bucket.go

@@ -903,7 +903,7 @@ func (b *Bucket) free() {
 	var tx = b.tx
 	b.forEachPageNode(func(p *common.Page, n *node, _ int) {
 		if p != nil {
-			tx.db.freelist.free(tx.meta.Txid(), p)
+			tx.db.freelist.Free(tx.meta.Txid(), p)
 		} else {
 			n.free()
 		}

bucket_test.go

@@ -419,16 +419,19 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) {
 
 	// Check more than an overflow's worth of pages are freed.
 	stats := db.Stats()
-	freePages := stats.FreePageN + stats.PendingPageN
-	if freePages <= 0xFFFF {
-		t.Fatalf("expected more than 0xFFFF free pages, got %v", freePages)
+	freeAndPending := stats.FreePageN + stats.PendingPageN
+	if freeAndPending <= 0xFFFF {
+		t.Fatalf("expected more than 0xFFFF free and pending pages, got %v", freeAndPending)
 	}
 
 	// Free page count should be preserved on reopen.
 	db.MustClose()
 	db.MustReopen()
-	if reopenFreePages := db.Stats().FreePageN; freePages != reopenFreePages {
-		t.Fatalf("expected %d free pages, got %+v", freePages, db.Stats())
+	if reopenFreePages := db.Stats().FreePageN; stats.FreePageN != reopenFreePages {
+		t.Fatalf("expected %d free pages, got %+v", stats.FreePageN, db.Stats())
+	}
+	if reopenPendingPages := db.Stats().PendingPageN; reopenPendingPages != 0 {
+		t.Fatalf("expected no pending pages, got %+v", db.Stats())
 	}
 }
 

cmd/bbolt/command_version.go

@@ -5,6 +5,7 @@ import (
 	"runtime"
 
 	"github.com/spf13/cobra"
+
 	"go.etcd.io/bbolt/version"
 )
 

concurrent_test.go

@@ -17,6 +17,8 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"go.etcd.io/bbolt/internal/freelist"
+
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 
@@ -235,9 +237,9 @@ func mustOpenDB(t *testing.T, dbPath string, o *bolt.Options) *bolt.DB {
 		o = bolt.DefaultOptions
 	}
 
-	freelistType := bolt.FreelistArrayType
-	if env := os.Getenv("TEST_FREELIST_TYPE"); env == string(bolt.FreelistMapType) {
-		freelistType = bolt.FreelistMapType
+	freelistType := freelist.FreelistArrayType
+	if env := os.Getenv("TEST_FREELIST_TYPE"); env == string(freelist.FreelistMapType) {
+		freelistType = freelist.FreelistMapType
 	}
 
 	o.FreelistType = freelistType
@@ -767,29 +769,29 @@ func TestConcurrentRepeatableRead(t *testing.T) {
 	testCases := []struct {
 		name           string
 		noFreelistSync bool
-		freelistType   bolt.FreelistType
+		freelistType   freelist.FreelistType
 	}{
 		// [array] freelist
 		{
 			name:           "sync array freelist",
 			noFreelistSync: false,
-			freelistType:   bolt.FreelistArrayType,
+			freelistType:   freelist.FreelistArrayType,
 		},
 		{
 			name:           "not sync array freelist",
 			noFreelistSync: true,
-			freelistType:   bolt.FreelistArrayType,
+			freelistType:   freelist.FreelistArrayType,
 		},
 		// [map] freelist
 		{
 			name:           "sync map freelist",
 			noFreelistSync: false,
-			freelistType:   bolt.FreelistMapType,
+			freelistType:   freelist.FreelistMapType,
 		},
 		{
 			name:           "not sync map freelist",
 			noFreelistSync: true,
-			freelistType:   bolt.FreelistMapType,
+			freelistType:   freelist.FreelistMapType,
 		},
 	}
 

db.go

@@ -13,25 +13,12 @@ import (
 
 	berrors "go.etcd.io/bbolt/errors"
 	"go.etcd.io/bbolt/internal/common"
+	fl "go.etcd.io/bbolt/internal/freelist"
 )
 
 // The time elapsed between consecutive file locking attempts.
 const flockRetryTimeout = 50 * time.Millisecond
 
-// FreelistType is the type of the freelist backend
-type FreelistType string
-
-// TODO(ahrtr): eventually we should (step by step)
-//  1. default to `FreelistMapType`;
-//  2. remove the `FreelistArrayType`, do not export `FreelistMapType`
-//     and remove field `FreelistType' from both `DB` and `Options`;
-const (
-	// FreelistArrayType indicates backend freelist type is array
-	FreelistArrayType = FreelistType("array")
-	// FreelistMapType indicates backend freelist type is hashmap
-	FreelistMapType = FreelistType("hashmap")
-)
-
 // DB represents a collection of buckets persisted to a file on disk.
 // All data access is performed through transactions which can be obtained through the DB.
 // All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
@@ -70,7 +57,7 @@ type DB struct {
 	// The alternative one is using hashmap, it is faster in almost all circumstances
 	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
 	// The default type is array
-	FreelistType FreelistType
+	FreelistType fl.FreelistType
 
 	// When true, skips the truncate call when growing the database.
 	// Setting this to true is only safe on non-ext3/ext4 systems.
@@ -134,8 +121,9 @@ type DB struct {
 	rwtx     *Tx
 	txs      []*Tx
 
-	freelist     *freelist
-	freelistLoad sync.Once
+	freelist           fl.Freelist
+	freelistSerializer fl.Serializable
+	freelistLoad       sync.Once
 
 	pagePool sync.Pool
 
@@ -190,6 +178,7 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) {
 	db.NoFreelistSync = options.NoFreelistSync
 	db.PreLoadFreelist = options.PreLoadFreelist
 	db.FreelistType = options.FreelistType
+	db.freelistSerializer = fl.Serializer{}
 	db.Mlock = options.Mlock
 
 	// Set default values for later DB operations.
@@ -416,15 +405,16 @@ func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
 // concurrent accesses being made to the freelist.
 func (db *DB) loadFreelist() {
 	db.freelistLoad.Do(func() {
-		db.freelist = newFreelist(db.FreelistType)
+		db.freelist = fl.NewFreelist(db.FreelistType)
 		if !db.hasSyncedFreelist() {
 			// Reconstruct free list by scanning the DB.
-			db.freelist.readIDs(db.freepages())
+			db.freelist.Init(db.freepages())
 		} else {
 			// Read free list from freelist page.
-			db.freelist.read(db.page(db.meta().Freelist()))
+			db.freelistSerializer.Read(db.freelist, db.page(db.meta().Freelist()))
 		}
-		db.stats.FreePageN = db.freelist.free_count()
+		db.stats.FreePageN = db.freelist.FreeCount()
+		db.stats.PendingPageN = db.freelist.PendingCount()
 	})
 }
 
@@ -854,14 +844,14 @@ func (db *DB) freePages() {
 		minid = db.txs[0].meta.Txid()
 	}
 	if minid > 0 {
-		db.freelist.release(minid - 1)
+		db.freelist.Release(minid - 1)
 	}
 	// Release unused txid extents.
 	for _, t := range db.txs {
-		db.freelist.releaseRange(minid, t.meta.Txid()-1)
+		db.freelist.ReleaseRange(minid, t.meta.Txid()-1)
 		minid = t.meta.Txid() + 1
 	}
-	db.freelist.releaseRange(minid, common.Txid(0xFFFFFFFFFFFFFFFF))
+	db.freelist.ReleaseRange(minid, common.Txid(0xFFFFFFFFFFFFFFFF))
 	// Any page both allocated and freed in an extent is safe to release.
 }
 
@@ -1176,7 +1166,7 @@ func (db *DB) allocate(txid common.Txid, count int) (*common.Page, error) {
 	p.SetOverflow(uint32(count - 1))
 
 	// Use pages from the freelist if they are available.
-	p.SetId(db.freelist.allocate(txid, count))
+	p.SetId(db.freelist.Allocate(txid, count))
 	if p.Id() != 0 {
 		return p, nil
 	}
@@ -1305,7 +1295,7 @@ type Options struct {
 	// The alternative one is using hashmap, it is faster in almost all circumstances
 	// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
 	// The default type is array
-	FreelistType FreelistType
+	FreelistType fl.FreelistType
 
 	// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
 	// grab a shared lock (UNIX).
@@ -1360,7 +1350,7 @@ func (o *Options) String() string {
 var DefaultOptions = &Options{
 	Timeout:      0,
 	NoGrowSync:   false,
-	FreelistType: FreelistArrayType,
+	FreelistType: fl.FreelistArrayType,
 }
 
 // Stats represents statistics about the database.

db_test.go

@@ -1053,8 +1053,8 @@ func TestDB_Stats(t *testing.T) {
 	stats := db.Stats()
 	if stats.TxStats.GetPageCount() != 2 {
 		t.Fatalf("unexpected TxStats.PageCount: %d", stats.TxStats.GetPageCount())
-	} else if stats.FreePageN != 0 {
-		t.Fatalf("unexpected FreePageN != 0: %d", stats.FreePageN)
+	} else if stats.FreePageN != 2 {
+		t.Fatalf("unexpected FreePageN != 2: %d", stats.FreePageN)
 	} else if stats.PendingPageN != 2 {
 		t.Fatalf("unexpected PendingPageN != 2: %d", stats.PendingPageN)
 	}