Skip to content

Commit 7e565c0

Browse files
tjungblutjungblu
committed
introduce a freelist interface
This introduces an interface for the freelist, splits it into two concrete implementations. fixes #773 Signed-off-by: Thomas Jungblut <[email protected]>
1 parent f8ffaee commit 7e565c0

21 files changed

+1013
-1058
lines changed

‎allocate_test.go

+4-3
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,13 @@ import (
44
"testing"
55

66
"go.etcd.io/bbolt/internal/common"
7+
"go.etcd.io/bbolt/internal/freelist"
78
)
89

910
func TestTx_allocatePageStats(t *testing.T) {
10-
f := newTestFreelist()
11+
f := freelist.NewFreelist(freelist.FreelistArrayType)
1112
ids := []common.Pgid{2, 3}
12-
f.readIDs(ids)
13+
f.Init(ids)
1314

1415
tx := &Tx{
1516
db: &DB{
@@ -22,7 +23,7 @@ func TestTx_allocatePageStats(t *testing.T) {
2223

2324
txStats := tx.Stats()
2425
prePageCnt := txStats.GetPageCount()
25-
allocateCnt := f.free_count()
26+
allocateCnt := f.FreeCount()
2627

2728
if _, err := tx.allocate(allocateCnt); err != nil {
2829
t.Fatal(err)

‎bucket.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -903,7 +903,7 @@ func (b *Bucket) free() {
903903
var tx = b.tx
904904
b.forEachPageNode(func(p *common.Page, n *node, _ int) {
905905
if p != nil {
906-
tx.db.freelist.free(tx.meta.Txid(), p)
906+
tx.db.freelist.Free(tx.meta.Txid(), p)
907907
} else {
908908
n.free()
909909
}

‎bucket_test.go

+8-5
Original file line numberDiff line numberDiff line change
@@ -419,16 +419,19 @@ func TestBucket_Delete_FreelistOverflow(t *testing.T) {
419419

420420
// Check more than an overflow's worth of pages are freed.
421421
stats := db.Stats()
422-
freePages := stats.FreePageN + stats.PendingPageN
423-
if freePages <= 0xFFFF {
424-
t.Fatalf("expected more than 0xFFFF free pages, got %v", freePages)
422+
freeAndPending := stats.FreePageN + stats.PendingPageN
423+
if freeAndPending <= 0xFFFF {
424+
t.Fatalf("expected more than 0xFFFF free and pending pages, got %v", freeAndPending)
425425
}
426426

427427
// Free page count should be preserved on reopen.
428428
db.MustClose()
429429
db.MustReopen()
430-
if reopenFreePages := db.Stats().FreePageN; freePages != reopenFreePages {
431-
t.Fatalf("expected %d free pages, got %+v", freePages, db.Stats())
430+
if reopenFreePages := db.Stats().FreePageN; stats.FreePageN != reopenFreePages {
431+
t.Fatalf("expected %d free pages, got %+v", stats.FreePageN, db.Stats())
432+
}
433+
if reopenPendingPages := db.Stats().PendingPageN; reopenPendingPages != 0 {
434+
t.Fatalf("expected no pending pages, got %+v", db.Stats())
432435
}
433436
}
434437

‎cmd/bbolt/command_version.go

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"runtime"
66

77
"github.com/spf13/cobra"
8+
89
"go.etcd.io/bbolt/version"
910
)
1011

‎concurrent_test.go

+10-8
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import (
1717
"time"
1818
"unicode/utf8"
1919

20+
"go.etcd.io/bbolt/internal/freelist"
21+
2022
"github.com/stretchr/testify/require"
2123
"golang.org/x/sync/errgroup"
2224

@@ -235,9 +237,9 @@ func mustOpenDB(t *testing.T, dbPath string, o *bolt.Options) *bolt.DB {
235237
o = bolt.DefaultOptions
236238
}
237239

238-
freelistType := bolt.FreelistArrayType
239-
if env := os.Getenv("TEST_FREELIST_TYPE"); env == string(bolt.FreelistMapType) {
240-
freelistType = bolt.FreelistMapType
240+
freelistType := freelist.FreelistArrayType
241+
if env := os.Getenv("TEST_FREELIST_TYPE"); env == string(freelist.FreelistMapType) {
242+
freelistType = freelist.FreelistMapType
241243
}
242244

243245
o.FreelistType = freelistType
@@ -767,29 +769,29 @@ func TestConcurrentRepeatableRead(t *testing.T) {
767769
testCases := []struct {
768770
name string
769771
noFreelistSync bool
770-
freelistType bolt.FreelistType
772+
freelistType freelist.FreelistType
771773
}{
772774
// [array] freelist
773775
{
774776
name: "sync array freelist",
775777
noFreelistSync: false,
776-
freelistType: bolt.FreelistArrayType,
778+
freelistType: freelist.FreelistArrayType,
777779
},
778780
{
779781
name: "not sync array freelist",
780782
noFreelistSync: true,
781-
freelistType: bolt.FreelistArrayType,
783+
freelistType: freelist.FreelistArrayType,
782784
},
783785
// [map] freelist
784786
{
785787
name: "sync map freelist",
786788
noFreelistSync: false,
787-
freelistType: bolt.FreelistMapType,
789+
freelistType: freelist.FreelistMapType,
788790
},
789791
{
790792
name: "not sync map freelist",
791793
noFreelistSync: true,
792-
freelistType: bolt.FreelistMapType,
794+
freelistType: freelist.FreelistMapType,
793795
},
794796
}
795797

‎db.go

+17-27
Original file line numberDiff line numberDiff line change
@@ -13,25 +13,12 @@ import (
1313

1414
berrors "go.etcd.io/bbolt/errors"
1515
"go.etcd.io/bbolt/internal/common"
16+
fl "go.etcd.io/bbolt/internal/freelist"
1617
)
1718

1819
// The time elapsed between consecutive file locking attempts.
1920
const flockRetryTimeout = 50 * time.Millisecond
2021

21-
// FreelistType is the type of the freelist backend
22-
type FreelistType string
23-
24-
// TODO(ahrtr): eventually we should (step by step)
25-
// 1. default to `FreelistMapType`;
26-
// 2. remove the `FreelistArrayType`, do not export `FreelistMapType`
27-
// and remove field `FreelistType' from both `DB` and `Options`;
28-
const (
29-
// FreelistArrayType indicates backend freelist type is array
30-
FreelistArrayType = FreelistType("array")
31-
// FreelistMapType indicates backend freelist type is hashmap
32-
FreelistMapType = FreelistType("hashmap")
33-
)
34-
3522
// DB represents a collection of buckets persisted to a file on disk.
3623
// All data access is performed through transactions which can be obtained through the DB.
3724
// All the functions on DB will return a ErrDatabaseNotOpen if accessed before Open() is called.
@@ -70,7 +57,7 @@ type DB struct {
7057
// The alternative one is using hashmap, it is faster in almost all circumstances
7158
// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
7259
// The default type is array
73-
FreelistType FreelistType
60+
FreelistType fl.FreelistType
7461

7562
// When true, skips the truncate call when growing the database.
7663
// Setting this to true is only safe on non-ext3/ext4 systems.
@@ -134,8 +121,9 @@ type DB struct {
134121
rwtx *Tx
135122
txs []*Tx
136123

137-
freelist *freelist
138-
freelistLoad sync.Once
124+
freelist fl.Freelist
125+
freelistSerializer fl.Serializable
126+
freelistLoad sync.Once
139127

140128
pagePool sync.Pool
141129

@@ -190,6 +178,7 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) {
190178
db.NoFreelistSync = options.NoFreelistSync
191179
db.PreLoadFreelist = options.PreLoadFreelist
192180
db.FreelistType = options.FreelistType
181+
db.freelistSerializer = fl.Serializer{}
193182
db.Mlock = options.Mlock
194183

195184
// Set default values for later DB operations.
@@ -416,15 +405,16 @@ func (db *DB) getPageSizeFromSecondMeta() (int, bool, error) {
416405
// concurrent accesses being made to the freelist.
417406
func (db *DB) loadFreelist() {
418407
db.freelistLoad.Do(func() {
419-
db.freelist = newFreelist(db.FreelistType)
408+
db.freelist = fl.NewFreelist(db.FreelistType)
420409
if !db.hasSyncedFreelist() {
421410
// Reconstruct free list by scanning the DB.
422-
db.freelist.readIDs(db.freepages())
411+
db.freelist.Init(db.freepages())
423412
} else {
424413
// Read free list from freelist page.
425-
db.freelist.read(db.page(db.meta().Freelist()))
414+
db.freelistSerializer.Read(db.freelist, db.page(db.meta().Freelist()))
426415
}
427-
db.stats.FreePageN = db.freelist.free_count()
416+
db.stats.FreePageN = db.freelist.FreeCount()
417+
db.stats.PendingPageN = db.freelist.PendingCount()
428418
})
429419
}
430420

@@ -854,14 +844,14 @@ func (db *DB) freePages() {
854844
minid = db.txs[0].meta.Txid()
855845
}
856846
if minid > 0 {
857-
db.freelist.release(minid - 1)
847+
db.freelist.Release(minid - 1)
858848
}
859849
// Release unused txid extents.
860850
for _, t := range db.txs {
861-
db.freelist.releaseRange(minid, t.meta.Txid()-1)
851+
db.freelist.ReleaseRange(minid, t.meta.Txid()-1)
862852
minid = t.meta.Txid() + 1
863853
}
864-
db.freelist.releaseRange(minid, common.Txid(0xFFFFFFFFFFFFFFFF))
854+
db.freelist.ReleaseRange(minid, common.Txid(0xFFFFFFFFFFFFFFFF))
865855
// Any page both allocated and freed in an extent is safe to release.
866856
}
867857

@@ -1176,7 +1166,7 @@ func (db *DB) allocate(txid common.Txid, count int) (*common.Page, error) {
11761166
p.SetOverflow(uint32(count - 1))
11771167

11781168
// Use pages from the freelist if they are available.
1179-
p.SetId(db.freelist.allocate(txid, count))
1169+
p.SetId(db.freelist.Allocate(txid, count))
11801170
if p.Id() != 0 {
11811171
return p, nil
11821172
}
@@ -1305,7 +1295,7 @@ type Options struct {
13051295
// The alternative one is using hashmap, it is faster in almost all circumstances
13061296
// but it doesn't guarantee that it offers the smallest page id available. In normal case it is safe.
13071297
// The default type is array
1308-
FreelistType FreelistType
1298+
FreelistType fl.FreelistType
13091299

13101300
// Open database in read-only mode. Uses flock(..., LOCK_SH |LOCK_NB) to
13111301
// grab a shared lock (UNIX).
@@ -1360,7 +1350,7 @@ func (o *Options) String() string {
13601350
var DefaultOptions = &Options{
13611351
Timeout: 0,
13621352
NoGrowSync: false,
1363-
FreelistType: FreelistArrayType,
1353+
FreelistType: fl.FreelistArrayType,
13641354
}
13651355

13661356
// Stats represents statistics about the database.

0 commit comments

Comments
 (0)