Skip to content

Commit 485fa74

Browse files
author
s-christoff
authored
Merge pull request #23 from hashicorp/transition
Add helper to go from BoltDB -> Bbolt
2 parents a7e3417 + c9a6551 commit 485fa74

File tree

6 files changed

+140
-5
lines changed

6 files changed

+140
-5
lines changed

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,4 @@ require (
66
github.com/boltdb/bolt v1.3.1
77
github.com/hashicorp/go-msgpack v0.5.5
88
github.com/hashicorp/raft v1.1.0
9-
)
9+
)

‎go.sum

+1-3
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,4 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
4040
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
4141
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
4242
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
43-
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
44-
golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed h1:uPxWBzB3+mlnjy9W58qY1j/cjyFjutgw/Vhan2zLy/A=
45-
golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
43+
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

‎v2/bolt_store.go

+68
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@ package raftboltdb
22

33
import (
44
"errors"
5+
"fmt"
6+
"os"
7+
"time"
58

9+
v1 "github.com/boltdb/bolt"
610
"github.com/hashicorp/raft"
711
"go.etcd.io/bbolt"
812
)
@@ -270,3 +274,67 @@ func (b *BoltStore) GetUint64(key []byte) (uint64, error) {
270274
func (b *BoltStore) Sync() error {
271275
return b.conn.Sync()
272276
}
277+
278+
// MigrateToV2 reads in the source file path of a BoltDB file
279+
// and outputs all the data migrated to a Bbolt destination file
280+
func MigrateToV2(source, destination string) (*BoltStore, error) {
281+
_, err := os.Stat(destination)
282+
if err == nil {
283+
return nil, fmt.Errorf("file exists in destination %v", destination)
284+
}
285+
286+
srcDb, err := v1.Open(source, dbFileMode, &v1.Options{
287+
ReadOnly: true,
288+
Timeout: 1 * time.Minute,
289+
})
290+
if err != nil {
291+
return nil, fmt.Errorf("failed opening source database: %v", err)
292+
}
293+
294+
//Start a connection to the source
295+
srctx, err := srcDb.Begin(false)
296+
if err != nil {
297+
return nil, fmt.Errorf("failed connecting to source database: %v", err)
298+
}
299+
defer srctx.Rollback()
300+
301+
//Create the destination
302+
destDb, err := New(Options{Path: destination})
303+
if err != nil {
304+
return nil, fmt.Errorf("failed creating destination database: %v", err)
305+
}
306+
//Start a connection to the new
307+
desttx, err := destDb.conn.Begin(true)
308+
if err != nil {
309+
destDb.Close()
310+
os.Remove(destination)
311+
return nil, fmt.Errorf("failed connecting to destination database: %v", err)
312+
}
313+
314+
defer desttx.Rollback()
315+
316+
//Loop over both old buckets and set them in the new
317+
buckets := [][]byte{dbConf, dbLogs}
318+
for _, b := range buckets {
319+
srcB := srctx.Bucket(b)
320+
destB := desttx.Bucket(b)
321+
err = srcB.ForEach(func(k, v []byte) error {
322+
return destB.Put(k, v)
323+
})
324+
if err != nil {
325+
destDb.Close()
326+
os.Remove(destination)
327+
return nil, fmt.Errorf("failed to copy %v bucket: %v", string(b), err)
328+
}
329+
}
330+
331+
//If the commit fails, clean up
332+
if err := desttx.Commit(); err != nil {
333+
destDb.Close()
334+
os.Remove(destination)
335+
return nil, fmt.Errorf("failed commiting data to destination: %v", err)
336+
}
337+
338+
return destDb, nil
339+
340+
}

‎v2/bolt_store_test.go

+58-1
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ import (
44
"bytes"
55
"io/ioutil"
66
"os"
7+
"path/filepath"
78
"reflect"
89
"testing"
910
"time"
1011

11-
"go.etcd.io/bbolt"
1212
"github.com/hashicorp/raft"
13+
v1 "github.com/hashicorp/raft-boltdb"
14+
"go.etcd.io/bbolt"
1315
)
1416

1517
func testBoltStore(t testing.TB) *BoltStore {
@@ -414,3 +416,58 @@ func TestBoltStore_SetUint64_GetUint64(t *testing.T) {
414416
t.Fatalf("bad: %v", val)
415417
}
416418
}
419+
420+
func TestBoltStore_MigrateToV2(t *testing.T) {
421+
422+
dir, err := ioutil.TempDir("", t.Name())
423+
if err != nil {
424+
t.Fatalf("err: %s", err)
425+
}
426+
defer os.RemoveAll(dir)
427+
428+
srcFile := filepath.Join(dir, "/sourcepath")
429+
destFile := filepath.Join(dir, "/destpath")
430+
431+
// Successfully creates and returns a store
432+
srcDb, err := v1.NewBoltStore(srcFile)
433+
if err != nil {
434+
t.Fatalf("failed creating source database: %s", err)
435+
}
436+
defer srcDb.Close()
437+
438+
// Set a mock raft log
439+
logs := []*raft.Log{
440+
testRaftLog(1, "log1"),
441+
testRaftLog(2, "log2"),
442+
testRaftLog(3, "log3"),
443+
}
444+
445+
//Store logs source
446+
if err := srcDb.StoreLogs(logs); err != nil {
447+
t.Fatalf("failed storing logs in source database: %s", err)
448+
}
449+
srcResult := new(raft.Log)
450+
if err := srcDb.GetLog(2, srcResult); err != nil {
451+
t.Fatalf("failed getting log from source database: %s", err)
452+
}
453+
454+
if err := srcDb.Close(); err != nil {
455+
t.Fatalf("failed closing source database: %s", err)
456+
}
457+
458+
destDb, err := MigrateToV2(srcFile, destFile)
459+
if err != nil {
460+
t.Fatalf("did not migrate successfully, err %v", err)
461+
}
462+
defer destDb.Close()
463+
464+
destResult := new(raft.Log)
465+
if err := destDb.GetLog(2, destResult); err != nil {
466+
t.Fatalf("failed getting log from destination database: %s", err)
467+
}
468+
469+
if !reflect.DeepEqual(srcResult, destResult) {
470+
t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", srcResult, destResult)
471+
}
472+
473+
}

‎v2/go.mod

+2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ module github.com/hashicorp/raft-boltdb/v2
33
go 1.12
44

55
require (
6+
github.com/boltdb/bolt v1.3.1
67
github.com/hashicorp/go-msgpack v0.5.5
78
github.com/hashicorp/raft v1.1.0
9+
github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea
810
go.etcd.io/bbolt v1.3.5
911
)

‎v2/go.sum

+10
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
22
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
33
github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
44
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
5+
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
6+
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
57
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
68
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
79
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
10+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
811
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
912
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
1013
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -15,25 +18,32 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
1518
github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
1619
github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
1720
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
21+
github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
1822
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
1923
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
2024
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
2125
github.com/hashicorp/raft v1.1.0 h1:qPMePEczgbkiQsqCsRfuHRqvDUO+zmAInDaD5ptXlq0=
2226
github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
27+
github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU=
28+
github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I=
2329
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
30+
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
2431
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
2532
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
33+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
2634
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
2735
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
2836
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
2937
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
3038
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
3139
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
3240
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
41+
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
3342
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
3443
github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
3544
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
3645
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
3746
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
3847
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
48+
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
3949
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

0 commit comments

Comments
 (0)