Skip to content

Commit 88d2b54

Browse files
mattsainsmattsains
authored
add support for data file size limit (#929)
* add support for data file size limit closes #928 Signed-off-by: Matthew Sainsbury <[email protected]> * respond to PR feedback Signed-off-by: Matthew Sainsbury <[email protected]> --------- Signed-off-by: Matthew Sainsbury <[email protected]>
1 parent 6e830d9 commit 88d2b54

File tree

5 files changed

+223
-5
lines changed

5 files changed

+223
-5
lines changed

‎bolt_windows.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,19 @@ func mmap(db *DB, sz int) error {
6767
var sizelo, sizehi uint32
6868

6969
if !db.readOnly {
70+
if db.MaxSize > 0 && sz > db.MaxSize {
71+
// The max size only limits future writes; however, we don’t block opening
72+
// and mapping the database if it already exceeds the limit.
73+
fileSize, err := db.fileSize()
74+
if err != nil {
75+
return fmt.Errorf("could not check existing db file size: %s", err)
76+
}
77+
78+
if sz > fileSize {
79+
return errors.ErrMaxSizeReached
80+
}
81+
}
82+
7083
// Truncate the database to the size of the mmap.
7184
if err := db.file.Truncate(int64(sz)); err != nil {
7285
return fmt.Errorf("truncate: %s", err)

‎db.go

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,12 @@ type DB struct {
110110
// of truncate() and fsync() when growing the data file.
111111
AllocSize int
112112

113+
// MaxSize is the maximum size (in bytes) allowed for the data file.
114+
// If a caller's attempt to add data results in the need to grow
115+
// the data file, an error will be returned and the data file will not grow.
116+
// <=0 means no limit.
117+
MaxSize int
118+
113119
// Mlock locks database file in memory when set to true.
114120
// It prevents major page faults, however used memory can't be reclaimed.
115121
//
@@ -191,6 +197,7 @@ func Open(path string, mode os.FileMode, options *Options) (db *DB, err error) {
191197
db.PreLoadFreelist = options.PreLoadFreelist
192198
db.FreelistType = options.FreelistType
193199
db.Mlock = options.Mlock
200+
db.MaxSize = options.MaxSize
194201

195202
// Set default values for later DB operations.
196203
db.MaxBatchSize = common.DefaultMaxBatchSize
@@ -1166,7 +1173,11 @@ func (db *DB) allocate(txid common.Txid, count int) (*common.Page, error) {
11661173
var minsz = int((p.Id()+common.Pgid(count))+1) * db.pageSize
11671174
if minsz >= db.datasz {
11681175
if err := db.mmap(minsz); err != nil {
1169-
return nil, fmt.Errorf("mmap allocate error: %s", err)
1176+
if err == berrors.ErrMaxSizeReached {
1177+
return nil, err
1178+
} else {
1179+
return nil, fmt.Errorf("mmap allocate error: %s", err)
1180+
}
11701181
}
11711182
}
11721183

@@ -1198,6 +1209,11 @@ func (db *DB) grow(sz int) error {
11981209
sz += db.AllocSize
11991210
}
12001211

1212+
if !db.readOnly && db.MaxSize > 0 && sz > db.MaxSize {
1213+
lg.Errorf("[GOOS: %s, GOARCH: %s] maximum db size reached, size: %d, db.MaxSize: %d", runtime.GOOS, runtime.GOARCH, sz, db.MaxSize)
1214+
return berrors.ErrMaxSizeReached
1215+
}
1216+
12011217
// Truncate and fsync to ensure file size metadata is flushed.
12021218
// https://github.com/boltdb/bolt/issues/284
12031219
if !db.NoGrowSync && !db.readOnly {
@@ -1320,6 +1336,9 @@ type Options struct {
13201336
// PageSize overrides the default OS page size.
13211337
PageSize int
13221338

1339+
// MaxSize sets the maximum size of the data file. <=0 means no maximum.
1340+
MaxSize int
1341+
13231342
// NoSync sets the initial value of DB.NoSync. Normally this can just be
13241343
// set directly on the DB itself when returned from Open(), but this option
13251344
// is useful in APIs which expose Options but not the underlying DB.
@@ -1343,8 +1362,8 @@ func (o *Options) String() string {
13431362
return "{}"
13441363
}
13451364

1346-
return fmt.Sprintf("{Timeout: %s, NoGrowSync: %t, NoFreelistSync: %t, PreLoadFreelist: %t, FreelistType: %s, ReadOnly: %t, MmapFlags: %x, InitialMmapSize: %d, PageSize: %d, NoSync: %t, OpenFile: %p, Mlock: %t, Logger: %p}",
1347-
o.Timeout, o.NoGrowSync, o.NoFreelistSync, o.PreLoadFreelist, o.FreelistType, o.ReadOnly, o.MmapFlags, o.InitialMmapSize, o.PageSize, o.NoSync, o.OpenFile, o.Mlock, o.Logger)
1365+
return fmt.Sprintf("{Timeout: %s, NoGrowSync: %t, NoFreelistSync: %t, PreLoadFreelist: %t, FreelistType: %s, ReadOnly: %t, MmapFlags: %x, InitialMmapSize: %d, PageSize: %d, MaxSize: %d, NoSync: %t, OpenFile: %p, Mlock: %t, Logger: %p}",
1366+
o.Timeout, o.NoGrowSync, o.NoFreelistSync, o.PreLoadFreelist, o.FreelistType, o.ReadOnly, o.MmapFlags, o.InitialMmapSize, o.PageSize, o.MaxSize, o.NoSync, o.OpenFile, o.Mlock, o.Logger)
13481367

13491368
}
13501369

‎db_test.go

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"os"
1212
"path/filepath"
1313
"reflect"
14+
"runtime"
1415
"strings"
1516
"sync"
1617
"testing"
@@ -1373,6 +1374,179 @@ func TestDBUnmap(t *testing.T) {
13731374
db.DB = nil
13741375
}
13751376

1377+
// Convenience function for inserting a bunch of keys with 1000 byte values
1378+
func fillDBWithKeys(db *btesting.DB, numKeys int) error {
1379+
return db.Fill([]byte("data"), 1, numKeys,
1380+
func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
1381+
func(tx int, k int) []byte { return make([]byte, 1000) },
1382+
)
1383+
}
1384+
1385+
// Creates a new database size, forces a specific allocation size jump, and fills it with the number of keys specified
1386+
func createFilledDB(t testing.TB, o *bolt.Options, allocSize int, numKeys int) *btesting.DB {
1387+
// Open a data file.
1388+
db := btesting.MustCreateDBWithOption(t, o)
1389+
db.AllocSize = allocSize
1390+
1391+
// Insert a reasonable amount of data below the max size.
1392+
err := db.Fill([]byte("data"), 1, numKeys,
1393+
func(tx int, k int) []byte { return []byte(fmt.Sprintf("%04d", k)) },
1394+
func(tx int, k int) []byte { return make([]byte, 1000) },
1395+
)
1396+
if err != nil {
1397+
t.Fatal(err)
1398+
}
1399+
return db
1400+
}
1401+
1402+
// Ensure that a database cannot exceed its maximum size
1403+
// https://github.com/etcd-io/bbolt/issues/928
1404+
func TestDB_MaxSizeNotExceeded(t *testing.T) {
1405+
testCases := []struct {
1406+
name string
1407+
options bolt.Options
1408+
}{
1409+
{
1410+
name: "Standard case",
1411+
options: bolt.Options{
1412+
MaxSize: 5 * 1024 * 1024, // 5 MiB
1413+
PageSize: 4096,
1414+
},
1415+
},
1416+
{
1417+
name: "NoGrowSync",
1418+
options: bolt.Options{
1419+
MaxSize: 5 * 1024 * 1024, // 5 MiB
1420+
PageSize: 4096,
1421+
NoGrowSync: true,
1422+
},
1423+
},
1424+
}
1425+
1426+
for _, testCase := range testCases {
1427+
t.Run(testCase.name, func(t *testing.T) {
1428+
db := createFilledDB(t,
1429+
&testCase.options,
1430+
4*1024*1024, // adjust allocation jumps to 4 MiB
1431+
2000,
1432+
)
1433+
1434+
path := db.Path()
1435+
1436+
// The data file should be 4 MiB now (expanded once from zero).
1437+
// It should have space for roughly 16 more entries before trying to grow
1438+
// Keep inserting until grow is required
1439+
err := fillDBWithKeys(db, 100)
1440+
assert.ErrorIs(t, err, berrors.ErrMaxSizeReached)
1441+
1442+
newSz := fileSize(path)
1443+
require.Greater(t, newSz, int64(0), "unexpected new file size: %d", newSz)
1444+
assert.LessOrEqual(t, newSz, int64(db.MaxSize), "The size of the data file should not exceed db.MaxSize")
1445+
1446+
err = db.Close()
1447+
require.NoError(t, err, "Closing the re-opened database should succeed")
1448+
})
1449+
}
1450+
}
1451+
1452+
// Ensure that opening a database that is beyond the maximum size succeeds
1453+
// The maximum size should only apply to growing the data file
1454+
// https://github.com/etcd-io/bbolt/issues/928
1455+
func TestDB_MaxSizeExceededCanOpen(t *testing.T) {
1456+
// Open a data file.
1457+
db := createFilledDB(t, nil, 4*1024*1024, 2000) // adjust allocation jumps to 4 MiB, fill with 2000, 1KB keys
1458+
path := db.Path()
1459+
1460+
// Insert a reasonable amount of data below the max size.
1461+
err := fillDBWithKeys(db, 2000)
1462+
require.NoError(t, err, "fillDbWithKeys should succeed")
1463+
1464+
err = db.Close()
1465+
require.NoError(t, err, "Close should succeed")
1466+
1467+
// The data file should be 4 MiB now (expanded once from zero).
1468+
minimumSizeForTest := int64(1024 * 1024)
1469+
newSz := fileSize(path)
1470+
require.GreaterOrEqual(t, newSz, minimumSizeForTest, "unexpected new file size: %d. Expected at least %d", newSz, minimumSizeForTest)
1471+
1472+
// Now try to re-open the database with an extremely small max size
1473+
t.Logf("Reopening bbolt DB at: %s", path)
1474+
db, err = btesting.OpenDBWithOption(t, path, &bolt.Options{
1475+
MaxSize: 1,
1476+
})
1477+
assert.NoError(t, err, "Should be able to open database bigger than MaxSize")
1478+
1479+
err = db.Close()
1480+
require.NoError(t, err, "Closing the re-opened database should succeed")
1481+
}
1482+
1483+
// Ensure that opening a database that is beyond the maximum size succeeds,
1484+
// even when InitialMmapSize is above the limit (mmaps should not affect file size)
1485+
// This test exists for platforms where Truncate should not be called during mmap
1486+
// https://github.com/etcd-io/bbolt/issues/928
1487+
func TestDB_MaxSizeExceededCanOpenWithHighMmap(t *testing.T) {
1488+
if runtime.GOOS == "windows" {
1489+
// In Windows, the file must be expanded to the mmap initial size,
1490+
// so this test doesn't run in Windows.
1491+
t.SkipNow()
1492+
}
1493+
1494+
// Open a data file.
1495+
db := createFilledDB(t, nil, 4*1024*1024, 2000) // adjust allocation jumps to 4 MiB, fill with 2000 1KB entries
1496+
path := db.Path()
1497+
1498+
err := db.Close()
1499+
require.NoError(t, err, "Close should succeed")
1500+
1501+
// The data file should be 4 MiB now (expanded once from zero).
1502+
minimumSizeForTest := int64(1024 * 1024)
1503+
newSz := fileSize(path)
1504+
require.GreaterOrEqual(t, newSz, minimumSizeForTest, "unexpected new file size: %d. Expected at least %d", newSz, minimumSizeForTest)
1505+
1506+
// Now try to re-open the database with an extremely small max size
1507+
t.Logf("Reopening bbolt DB at: %s", path)
1508+
db, err = btesting.OpenDBWithOption(t, path, &bolt.Options{
1509+
MaxSize: 1,
1510+
InitialMmapSize: int(minimumSizeForTest) * 2,
1511+
})
1512+
assert.NoError(t, err, "Should be able to open database bigger than MaxSize when InitialMmapSize set high")
1513+
1514+
err = db.Close()
1515+
require.NoError(t, err, "Closing the re-opened database should succeed")
1516+
}
1517+
1518+
// Ensure that when InitialMmapSize is above the limit, opening a database
1519+
// that is beyond the maximum size fails in Windows.
1520+
// In Windows, the file must be expanded to the mmap initial size.
1521+
// https://github.com/etcd-io/bbolt/issues/928
1522+
func TestDB_MaxSizeExceededDoesNotGrow(t *testing.T) {
1523+
if runtime.GOOS != "windows" {
1524+
// This test is only relevant on Windows
1525+
t.SkipNow()
1526+
}
1527+
1528+
// Open a data file.
1529+
db := createFilledDB(t, nil, 4*1024*1024, 2000) // adjust allocation jumps to 4 MiB, fill with 2000 1KB entries
1530+
path := db.Path()
1531+
1532+
err := db.Close()
1533+
require.NoError(t, err, "Close should succeed")
1534+
1535+
// The data file should be 4 MiB now (expanded once from zero).
1536+
minimumSizeForTest := int64(1024 * 1024)
1537+
newSz := fileSize(path)
1538+
assert.GreaterOrEqual(t, newSz, minimumSizeForTest, "unexpected new file size: %d. Expected at least %d", newSz, minimumSizeForTest)
1539+
1540+
// Now try to re-open the database with an extremely small max size and
1541+
// an initial mmap size to be greater than the actual file size, forcing an illegal grow on open
1542+
t.Logf("Reopening bbolt DB at: %s", path)
1543+
_, err = btesting.OpenDBWithOption(t, path, &bolt.Options{
1544+
MaxSize: 1,
1545+
InitialMmapSize: int(newSz) * 2,
1546+
})
1547+
assert.Error(t, err, "Opening the DB with InitialMmapSize > MaxSize should cause an error on Windows")
1548+
}
1549+
13761550
func ExampleDB_Update() {
13771551
// Open the database.
13781552
db, err := bolt.Open(tempfile(), 0600, nil)

‎errors/errors.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,9 @@ var (
6969
// ErrValueTooLarge is returned when inserting a value that is larger than MaxValueSize.
7070
ErrValueTooLarge = errors.New("value too large")
7171

72+
// ErrMaxSizeReached is returned when the configured maximum size of the data file is reached.
73+
ErrMaxSizeReached = errors.New("database reached maximum size")
74+
7275
// ErrIncompatibleValue is returned when trying to create or delete a bucket
7376
// on an existing non-bucket key or when trying to create or delete a
7477
// non-bucket key on an existing bucket key.

‎internal/btesting/btesting.go

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ func MustCreateDBWithOption(t testing.TB, o *bolt.Options) *DB {
4444
}
4545

4646
func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB {
47+
db, err := OpenDBWithOption(t, f, o)
48+
require.NoError(t, err)
49+
require.NotNil(t, db)
50+
return db
51+
}
52+
53+
func OpenDBWithOption(t testing.TB, f string, o *bolt.Options) (*DB, error) {
4754
t.Logf("Opening bbolt DB at: %s", f)
4855
if o == nil {
4956
o = bolt.DefaultOptions
@@ -57,7 +64,9 @@ func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB {
5764
o.FreelistType = freelistType
5865

5966
db, err := bolt.Open(f, 0600, o)
60-
require.NoError(t, err)
67+
if err != nil {
68+
return nil, err
69+
}
6170
resDB := &DB{
6271
DB: db,
6372
f: f,
@@ -66,7 +75,7 @@ func MustOpenDBWithOption(t testing.TB, f string, o *bolt.Options) *DB {
6675
}
6776
resDB.strictModeEnabledDefault()
6877
t.Cleanup(resDB.PostTestCleanup)
69-
return resDB
78+
return resDB, nil
7079
}
7180

7281
func (db *DB) PostTestCleanup() {

0 commit comments

Comments
 (0)