From 8b3ae8319b82eb06f6124acb50ecb45a48ead41e Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Tue, 20 Apr 2021 11:20:32 -0500 Subject: [PATCH 1/7] it worked? --- bolt_store.go | 24 +++++++------- go.mod | 1 + go.sum | 7 +++- v2/bolt_store.go | 33 +++++++++++++++++-- v2/bolt_store_test.go | 76 ++++++++++++++++++++++++++++++++++++++++++- v2/go.mod | 1 + v2/go.sum | 4 +++ 7 files changed, 130 insertions(+), 16 deletions(-) diff --git a/bolt_store.go b/bolt_store.go index 66b75bc..8e3efc6 100644 --- a/bolt_store.go +++ b/bolt_store.go @@ -27,7 +27,7 @@ var ( // a LogStore and StableStore. type BoltStore struct { // conn is the underlying handle to the db. - conn *bolt.DB + Conn *bolt.DB // The path to the Bolt database file path string @@ -71,7 +71,7 @@ func New(options Options) (*BoltStore, error) { // Create the new store store := &BoltStore{ - conn: handle, + Conn: handle, path: options.Path, } @@ -88,7 +88,7 @@ func New(options Options) (*BoltStore, error) { // initialize is used to set up all of the buckets. func (b *BoltStore) initialize() error { - tx, err := b.conn.Begin(true) + tx, err := b.Conn.Begin(true) if err != nil { return err } @@ -107,12 +107,12 @@ func (b *BoltStore) initialize() error { // Close is used to gracefully close the DB connection. func (b *BoltStore) Close() error { - return b.conn.Close() + return b.Conn.Close() } // FirstIndex returns the first known index from the Raft log. func (b *BoltStore) FirstIndex() (uint64, error) { - tx, err := b.conn.Begin(false) + tx, err := b.Conn.Begin(false) if err != nil { return 0, err } @@ -128,7 +128,7 @@ func (b *BoltStore) FirstIndex() (uint64, error) { // LastIndex returns the last known index from the Raft log. func (b *BoltStore) LastIndex() (uint64, error) { - tx, err := b.conn.Begin(false) + tx, err := b.Conn.Begin(false) if err != nil { return 0, err } @@ -144,7 +144,7 @@ func (b *BoltStore) LastIndex() (uint64, error) { // GetLog is used to retrieve a log from BoltDB at a given index. func (b *BoltStore) GetLog(idx uint64, log *raft.Log) error { - tx, err := b.conn.Begin(false) + tx, err := b.Conn.Begin(false) if err != nil { return err } @@ -166,7 +166,7 @@ func (b *BoltStore) StoreLog(log *raft.Log) error { // StoreLogs is used to store a set of raft logs func (b *BoltStore) StoreLogs(logs []*raft.Log) error { - tx, err := b.conn.Begin(true) + tx, err := b.Conn.Begin(true) if err != nil { return err } @@ -191,7 +191,7 @@ func (b *BoltStore) StoreLogs(logs []*raft.Log) error { func (b *BoltStore) DeleteRange(min, max uint64) error { minKey := uint64ToBytes(min) - tx, err := b.conn.Begin(true) + tx, err := b.Conn.Begin(true) if err != nil { return err } @@ -215,7 +215,7 @@ func (b *BoltStore) DeleteRange(min, max uint64) error { // Set is used to set a key/value set outside of the raft log func (b *BoltStore) Set(k, v []byte) error { - tx, err := b.conn.Begin(true) + tx, err := b.Conn.Begin(true) if err != nil { return err } @@ -231,7 +231,7 @@ func (b *BoltStore) Set(k, v []byte) error { // Get is used to retrieve a value from the k/v store by key func (b *BoltStore) Get(k []byte) ([]byte, error) { - tx, err := b.conn.Begin(false) + tx, err := b.Conn.Begin(false) if err != nil { return nil, err } @@ -264,5 +264,5 @@ func (b *BoltStore) GetUint64(key []byte) (uint64, error) { // under normal operation unless NoSync is enabled, in which this forces the // database file to sync against the disk. func (b *BoltStore) Sync() error { - return b.conn.Sync() + return b.Conn.Sync() } diff --git a/go.mod b/go.mod index 3ae4d9d..2b55c38 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 + github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210409134258-03c10cc3d4ea // indirect ) diff --git a/go.sum b/go.sum index b34e552..52a189e 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCO github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/raft v1.1.0 h1:qPMePEczgbkiQsqCsRfuHRqvDUO+zmAInDaD5ptXlq0= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210409134258-03c10cc3d4ea h1:pXD01QLdHmn4Ij82g1vksWbZXwSH6il7Svrm/rdUk18= +github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210409134258-03c10cc3d4ea/go.mod h1:kiPs9g148eLShc2TYagUAyKDnD+dH9U+CQKsXzlY9xo= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -39,7 +41,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed h1:uPxWBzB3+mlnjy9W58qY1j/cjyFjutgw/Vhan2zLy/A= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= \ No newline at end of file +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/v2/bolt_store.go b/v2/bolt_store.go index 0e365ee..a44a105 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/hashicorp/raft" + v1 "github.com/hashicorp/raft-boltdb" "go.etcd.io/bbolt" ) @@ -78,7 +79,7 @@ func New(options Options) (*BoltStore, error) { // If the store was opened read-only, don't try and create buckets if !options.readOnly() { // Set up our buckets - if err := store.initialize(); err != nil { + if err := store.Initialize(); err != nil { store.Close() return nil, err } @@ -87,7 +88,7 @@ func New(options Options) (*BoltStore, error) { } // initialize is used to set up all of the buckets. -func (b *BoltStore) initialize() error { +func (b *BoltStore) Initialize() error { tx, err := b.conn.Begin(true) if err != nil { return err @@ -266,3 +267,31 @@ func (b *BoltStore) GetUint64(key []byte) (uint64, error) { func (b *BoltStore) Sync() error { return b.conn.Sync() } + +func Transition(old *v1.BoltStore, path string) (*BoltStore, error) { + + //Create the bbolt log store + newbolt, err := New(Options{Path: path}) + + //Start a connection to the old + oldtx, err := old.Conn.Begin(false) + if err != nil { + return newbolt, err + } + + //Grab the old buckets + confBucket := oldtx.Bucket([]byte(dbConf)) + logBucket := oldtx.Bucket([]byte(dbLogs)) + + //Loop over both old buckets and set them in the new + confBucket.ForEach(func(k, v []byte) error { + return newbolt.Set(k, v) + }) + + logBucket.ForEach(func(k, v []byte) error { + return newbolt.Set(k, v) + }) + + return newbolt, nil + +} diff --git a/v2/bolt_store_test.go b/v2/bolt_store_test.go index 737efc2..567bd06 100644 --- a/v2/bolt_store_test.go +++ b/v2/bolt_store_test.go @@ -8,8 +8,9 @@ import ( "testing" "time" - "go.etcd.io/bbolt" "github.com/hashicorp/raft" + v1 "github.com/hashicorp/raft-boltdb" + "go.etcd.io/bbolt" ) func testBoltStore(t testing.TB) *BoltStore { @@ -414,3 +415,76 @@ func TestBoltStore_SetUint64_GetUint64(t *testing.T) { t.Fatalf("bad: %v", val) } } + +func TestBoltStore_TransitionBbolt(t *testing.T) { + + //Create BoltDB + fh, err := ioutil.TempFile("", "bolt") + if err != nil { + t.Fatalf("err: %s", err) + } + os.Remove(fh.Name()) + defer os.Remove(fh.Name()) + + // Successfully creates and returns a store + oldstore, err := v1.NewBoltStore(fh.Name()) + if err != nil { + t.Fatalf("err: %s", err) + } + + // Ensure the file was created + if oldstore.Options.Path != fh.Name() { + t.Fatalf("unexpected file path %q", oldstore.Options.Path) + } + if _, err := os.Stat(fh.Name()); err != nil { + t.Fatalf("err: %s", err) + } + + log := new(raft.Log) + + // Set a mock raft log + logs := []*raft.Log{ + testRaftLog(1, "log1"), + testRaftLog(2, "log2"), + testRaftLog(3, "log3"), + } + if err := oldstore.StoreLogs(logs); err != nil { + t.Fatalf("bad: %s", err) + } + + // Should return the proper log + oldresult := new(raft.Log) + if err := oldstore.GetLog(2, oldresult); err != nil { + t.Fatalf("err: %s", err) + } + if !reflect.DeepEqual(log, logs[1]) { + t.Fatalf("bad: %#v", log) + } + + // Close the store so we can open again + if err := oldstore.Close(); err != nil { + t.Fatalf("err: %s", err) + } + + //Send to func + newstore, err := Transition(oldstore) + if err != nil { + t.Fatalf("did not transition successfully, err %v", err) + } + + if err := newstore.StoreLogs(logs); err != nil { + t.Fatalf("bad: %s", err) + } + + // Should return the proper log + newresult := new(raft.Log) + if err := newstore.GetLog(2, newresult); err != nil { + t.Fatalf("err: %s", err) + } + + //Comapre + if !reflect.DeepEqual(oldresult, newresult) { + t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", oldresult, newresult) + } + +} diff --git a/v2/go.mod b/v2/go.mod index 2b589b6..5cbf8b3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -5,5 +5,6 @@ go 1.12 require ( github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 + github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea go.etcd.io/bbolt v1.3.5 ) diff --git a/v2/go.sum b/v2/go.sum index 4ca5a0b..9339d32 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -2,6 +2,8 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -20,6 +22,8 @@ github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCO github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/raft v1.1.0 h1:qPMePEczgbkiQsqCsRfuHRqvDUO+zmAInDaD5ptXlq0= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= +github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= From faa9e35ca95b9a2dee6745f1078781832dcb06a8 Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Tue, 20 Apr 2021 11:59:51 -0500 Subject: [PATCH 2/7] add accessor func and put kvs in proper buckets --- bolt_store.go | 30 ++++++++++++++++++------------ go.mod | 1 - v2/bolt_store.go | 19 ++++++++++++++++--- v2/bolt_store_test.go | 20 +++++--------------- 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/bolt_store.go b/bolt_store.go index 8e3efc6..4247dd4 100644 --- a/bolt_store.go +++ b/bolt_store.go @@ -27,7 +27,7 @@ var ( // a LogStore and StableStore. type BoltStore struct { // conn is the underlying handle to the db. - Conn *bolt.DB + conn *bolt.DB // The path to the Bolt database file path string @@ -71,7 +71,7 @@ func New(options Options) (*BoltStore, error) { // Create the new store store := &BoltStore{ - Conn: handle, + conn: handle, path: options.Path, } @@ -88,7 +88,7 @@ func New(options Options) (*BoltStore, error) { // initialize is used to set up all of the buckets. func (b *BoltStore) initialize() error { - tx, err := b.Conn.Begin(true) + tx, err := b.conn.Begin(true) if err != nil { return err } @@ -107,12 +107,12 @@ func (b *BoltStore) initialize() error { // Close is used to gracefully close the DB connection. func (b *BoltStore) Close() error { - return b.Conn.Close() + return b.conn.Close() } // FirstIndex returns the first known index from the Raft log. func (b *BoltStore) FirstIndex() (uint64, error) { - tx, err := b.Conn.Begin(false) + tx, err := b.conn.Begin(false) if err != nil { return 0, err } @@ -128,7 +128,7 @@ func (b *BoltStore) FirstIndex() (uint64, error) { // LastIndex returns the last known index from the Raft log. func (b *BoltStore) LastIndex() (uint64, error) { - tx, err := b.Conn.Begin(false) + tx, err := b.conn.Begin(false) if err != nil { return 0, err } @@ -144,7 +144,7 @@ func (b *BoltStore) LastIndex() (uint64, error) { // GetLog is used to retrieve a log from BoltDB at a given index. func (b *BoltStore) GetLog(idx uint64, log *raft.Log) error { - tx, err := b.Conn.Begin(false) + tx, err := b.conn.Begin(false) if err != nil { return err } @@ -166,7 +166,7 @@ func (b *BoltStore) StoreLog(log *raft.Log) error { // StoreLogs is used to store a set of raft logs func (b *BoltStore) StoreLogs(logs []*raft.Log) error { - tx, err := b.Conn.Begin(true) + tx, err := b.conn.Begin(true) if err != nil { return err } @@ -191,7 +191,7 @@ func (b *BoltStore) StoreLogs(logs []*raft.Log) error { func (b *BoltStore) DeleteRange(min, max uint64) error { minKey := uint64ToBytes(min) - tx, err := b.Conn.Begin(true) + tx, err := b.conn.Begin(true) if err != nil { return err } @@ -215,7 +215,7 @@ func (b *BoltStore) DeleteRange(min, max uint64) error { // Set is used to set a key/value set outside of the raft log func (b *BoltStore) Set(k, v []byte) error { - tx, err := b.Conn.Begin(true) + tx, err := b.conn.Begin(true) if err != nil { return err } @@ -231,7 +231,7 @@ func (b *BoltStore) Set(k, v []byte) error { // Get is used to retrieve a value from the k/v store by key func (b *BoltStore) Get(k []byte) ([]byte, error) { - tx, err := b.Conn.Begin(false) + tx, err := b.conn.Begin(false) if err != nil { return nil, err } @@ -264,5 +264,11 @@ func (b *BoltStore) GetUint64(key []byte) (uint64, error) { // under normal operation unless NoSync is enabled, in which this forces the // database file to sync against the disk. func (b *BoltStore) Sync() error { - return b.Conn.Sync() + return b.conn.Sync() +} + +// InternalUseOnlyAccessBoltDB is used for the v2/transition function +// WARNING: THIS IS ONLY TO BE USED BY THAT FUNCTION +func (b *BoltStore) InternalUseOnlyAccessBoltDB() *bolt.DB { + return b.conn } diff --git a/go.mod b/go.mod index 2b55c38..3ae4d9d 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,4 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 - github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210409134258-03c10cc3d4ea // indirect ) diff --git a/v2/bolt_store.go b/v2/bolt_store.go index a44a105..d283ad7 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -268,13 +268,18 @@ func (b *BoltStore) Sync() error { return b.conn.Sync() } +// Transition takes a BoltDB and a path, and outputs a +// Bbolt and generates it at that path func Transition(old *v1.BoltStore, path string) (*BoltStore, error) { //Create the bbolt log store newbolt, err := New(Options{Path: path}) + //Grab connection to the old store + oldconn := old.InternalUseOnlyAccessBoltDB() + //Start a connection to the old - oldtx, err := old.Conn.Begin(false) + oldtx, err := oldconn.Begin(false) if err != nil { return newbolt, err } @@ -283,13 +288,21 @@ func Transition(old *v1.BoltStore, path string) (*BoltStore, error) { confBucket := oldtx.Bucket([]byte(dbConf)) logBucket := oldtx.Bucket([]byte(dbLogs)) + //Start a connection to the new + newtx, err := newbolt.conn.Begin(false) + if err != nil { + return newbolt, err + } + //Loop over both old buckets and set them in the new confBucket.ForEach(func(k, v []byte) error { - return newbolt.Set(k, v) + bucket := newtx.Bucket(dbConf) + return bucket.Put(k, v) }) logBucket.ForEach(func(k, v []byte) error { - return newbolt.Set(k, v) + bucket := newtx.Bucket(dbLogs) + return bucket.Put(k, v) }) return newbolt, nil diff --git a/v2/bolt_store_test.go b/v2/bolt_store_test.go index 567bd06..dc87ae4 100644 --- a/v2/bolt_store_test.go +++ b/v2/bolt_store_test.go @@ -432,22 +432,15 @@ func TestBoltStore_TransitionBbolt(t *testing.T) { t.Fatalf("err: %s", err) } - // Ensure the file was created - if oldstore.Options.Path != fh.Name() { - t.Fatalf("unexpected file path %q", oldstore.Options.Path) - } - if _, err := os.Stat(fh.Name()); err != nil { - t.Fatalf("err: %s", err) - } - - log := new(raft.Log) - // Set a mock raft log + log := new(raft.Log) logs := []*raft.Log{ testRaftLog(1, "log1"), testRaftLog(2, "log2"), testRaftLog(3, "log3"), } + + //Store logs old if err := oldstore.StoreLogs(logs); err != nil { t.Fatalf("bad: %s", err) } @@ -457,21 +450,18 @@ func TestBoltStore_TransitionBbolt(t *testing.T) { if err := oldstore.GetLog(2, oldresult); err != nil { t.Fatalf("err: %s", err) } - if !reflect.DeepEqual(log, logs[1]) { - t.Fatalf("bad: %#v", log) - } - // Close the store so we can open again if err := oldstore.Close(); err != nil { t.Fatalf("err: %s", err) } //Send to func - newstore, err := Transition(oldstore) + newstore, err := Transition(oldstore, fh.Name()) if err != nil { t.Fatalf("did not transition successfully, err %v", err) } + //Store same logs in new if err := newstore.StoreLogs(logs); err != nil { t.Fatalf("bad: %s", err) } From a4afc141e48317640b670b1efc60eb3da7f39412 Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Tue, 20 Apr 2021 12:03:42 -0500 Subject: [PATCH 3/7] minor mistake --- v2/bolt_store.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/bolt_store.go b/v2/bolt_store.go index d283ad7..703ed35 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -79,7 +79,7 @@ func New(options Options) (*BoltStore, error) { // If the store was opened read-only, don't try and create buckets if !options.readOnly() { // Set up our buckets - if err := store.Initialize(); err != nil { + if err := store.initialize(); err != nil { store.Close() return nil, err } @@ -88,7 +88,7 @@ func New(options Options) (*BoltStore, error) { } // initialize is used to set up all of the buckets. -func (b *BoltStore) Initialize() error { +func (b *BoltStore) initialize() error { tx, err := b.conn.Begin(true) if err != nil { return err From 7935e8212722f40e47e7fcfe9ff1deb21f739f6d Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Wed, 21 Apr 2021 11:36:41 -0500 Subject: [PATCH 4/7] cleaner and passing --- bolt_store.go | 6 ---- v2/bolt_store.go | 80 ++++++++++++++++++++++++++++--------------- v2/bolt_store_test.go | 60 +++++++++++++++----------------- v2/go.mod | 2 ++ 4 files changed, 82 insertions(+), 66 deletions(-) diff --git a/bolt_store.go b/bolt_store.go index 4247dd4..66b75bc 100644 --- a/bolt_store.go +++ b/bolt_store.go @@ -266,9 +266,3 @@ func (b *BoltStore) GetUint64(key []byte) (uint64, error) { func (b *BoltStore) Sync() error { return b.conn.Sync() } - -// InternalUseOnlyAccessBoltDB is used for the v2/transition function -// WARNING: THIS IS ONLY TO BE USED BY THAT FUNCTION -func (b *BoltStore) InternalUseOnlyAccessBoltDB() *bolt.DB { - return b.conn -} diff --git a/v2/bolt_store.go b/v2/bolt_store.go index 703ed35..b935ca0 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -2,9 +2,12 @@ package raftboltdb import ( "errors" + "fmt" + "os" + "time" + boltdbbolt "github.com/boltdb/bolt" "github.com/hashicorp/raft" - v1 "github.com/hashicorp/raft-boltdb" "go.etcd.io/bbolt" ) @@ -268,43 +271,66 @@ func (b *BoltStore) Sync() error { return b.conn.Sync() } -// Transition takes a BoltDB and a path, and outputs a -// Bbolt and generates it at that path -func Transition(old *v1.BoltStore, path string) (*BoltStore, error) { - - //Create the bbolt log store - newbolt, err := New(Options{Path: path}) - - //Grab connection to the old store - oldconn := old.InternalUseOnlyAccessBoltDB() +// MigratetoV2 reads in the source file path of a BoltDB file +// and outputs all the data migrated to a Bbolt destination file +func MigratetoV2(source, destination string) (*BoltStore, error) { + _, err := os.Stat(destination) + if err == nil { + return nil, fmt.Errorf("file exists in destination %v", destination) + } - //Start a connection to the old - oldtx, err := oldconn.Begin(false) + sourceDb, err := boltdbbolt.Open(source, dbFileMode, &boltdbbolt.Options{ + ReadOnly: true, + Timeout: 1 * time.Minute, + }) if err != nil { - return newbolt, err + return nil, fmt.Errorf("failed opening source database: %v", err) } - //Grab the old buckets - confBucket := oldtx.Bucket([]byte(dbConf)) - logBucket := oldtx.Bucket([]byte(dbLogs)) + //Start a connection to the source + sourcetx, err := sourceDb.Begin(false) + if err != nil { + return nil, fmt.Errorf("failed connecting to source database: %v", err) + } + defer sourcetx.Rollback() + //Create the destination + destDb, err := New(Options{Path: destination}) + if err != nil { + return nil, fmt.Errorf("failed creating destination database: %v", err) + } //Start a connection to the new - newtx, err := newbolt.conn.Begin(false) + desttx, err := destDb.conn.Begin(true) if err != nil { - return newbolt, err + destDb.Close() + os.Remove(destination) + return nil, fmt.Errorf("failed connecting to destination database: %v", err) } + defer desttx.Rollback() + //Loop over both old buckets and set them in the new - confBucket.ForEach(func(k, v []byte) error { - bucket := newtx.Bucket(dbConf) - return bucket.Put(k, v) - }) + buckets := [][]byte{dbConf, dbLogs} + for _, b := range buckets { + sourceB := sourcetx.Bucket(b) + destB := desttx.Bucket(b) + err = sourceB.ForEach(func(k, v []byte) error { + return destB.Put(k, v) + }) + if err != nil { + destDb.Close() + os.Remove(destination) + return nil, fmt.Errorf("failed to copy %v bucket: %v", string(b), err) + } + } - logBucket.ForEach(func(k, v []byte) error { - bucket := newtx.Bucket(dbLogs) - return bucket.Put(k, v) - }) + //If the commit fails, clean up + if err := desttx.Commit(); err != nil { + destDb.Close() + os.Remove(destination) + return nil, fmt.Errorf("failed commiting data to destination: %v", err) + } - return newbolt, nil + return destDb, nil } diff --git a/v2/bolt_store_test.go b/v2/bolt_store_test.go index dc87ae4..617a961 100644 --- a/v2/bolt_store_test.go +++ b/v2/bolt_store_test.go @@ -4,6 +4,7 @@ import ( "bytes" "io/ioutil" "os" + "path/filepath" "reflect" "testing" "time" @@ -416,65 +417,58 @@ func TestBoltStore_SetUint64_GetUint64(t *testing.T) { } } -func TestBoltStore_TransitionBbolt(t *testing.T) { +func TestBoltStore_MigratetoV2(t *testing.T) { - //Create BoltDB - fh, err := ioutil.TempFile("", "bolt") + dir, err := ioutil.TempDir("", t.Name()) if err != nil { t.Fatalf("err: %s", err) } - os.Remove(fh.Name()) - defer os.Remove(fh.Name()) + defer os.RemoveAll(dir) + + var sourceFile, destFile string + sourceFile = filepath.Join(dir, "/sourcepath") + destFile = filepath.Join(dir, "/destpath") // Successfully creates and returns a store - oldstore, err := v1.NewBoltStore(fh.Name()) + sourceDb, err := v1.NewBoltStore(sourceFile) if err != nil { - t.Fatalf("err: %s", err) + t.Fatalf("failed creating source database: %s", err) } + defer sourceDb.Close() // Set a mock raft log - log := new(raft.Log) logs := []*raft.Log{ testRaftLog(1, "log1"), testRaftLog(2, "log2"), testRaftLog(3, "log3"), } - //Store logs old - if err := oldstore.StoreLogs(logs); err != nil { - t.Fatalf("bad: %s", err) + //Store logs source + if err := sourceDb.StoreLogs(logs); err != nil { + t.Fatalf("failed storing logs in source database: %s", err) } - - // Should return the proper log - oldresult := new(raft.Log) - if err := oldstore.GetLog(2, oldresult); err != nil { - t.Fatalf("err: %s", err) + sourceResult := new(raft.Log) + if err := sourceDb.GetLog(2, sourceResult); err != nil { + t.Fatalf("failed getting log from source database: %s", err) } - if err := oldstore.Close(); err != nil { - t.Fatalf("err: %s", err) + if err := sourceDb.Close(); err != nil { + t.Fatalf("failed closing source database: %s", err) } - //Send to func - newstore, err := Transition(oldstore, fh.Name()) + destDb, err := MigratetoV2(sourceFile, destFile) if err != nil { - t.Fatalf("did not transition successfully, err %v", err) - } - - //Store same logs in new - if err := newstore.StoreLogs(logs); err != nil { - t.Fatalf("bad: %s", err) + t.Fatalf("did not migrate successfully, err %v", err) } + defer destDb.Close() - // Should return the proper log - newresult := new(raft.Log) - if err := newstore.GetLog(2, newresult); err != nil { - t.Fatalf("err: %s", err) + destResult := new(raft.Log) + if err := destDb.GetLog(2, destResult); err != nil { + t.Fatalf("failed getting log from destination database: %s", err) } - //Comapre - if !reflect.DeepEqual(oldresult, newresult) { - t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", oldresult, newresult) + if !reflect.DeepEqual(sourceResult, destResult) { + t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", sourceResult, destResult) } } diff --git a/v2/go.mod b/v2/go.mod index 5cbf8b3..ae26313 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -3,8 +3,10 @@ module github.com/hashicorp/raft-boltdb/v2 go 1.12 require ( + github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea go.etcd.io/bbolt v1.3.5 ) + From fedb9bb4d715787bdf57228f470643a6096aa22c Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Wed, 21 Apr 2021 11:38:46 -0500 Subject: [PATCH 5/7] tidy the go mod --- go.mod | 1 + go.sum | 9 ++------- v2/go.mod | 1 - v2/go.sum | 6 ++++++ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 3ae4d9d..a1e1d62 100644 --- a/go.mod +++ b/go.mod @@ -6,4 +6,5 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 + golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 // indirect ) diff --git a/go.sum b/go.sum index 52a189e..02d1658 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCO github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/raft v1.1.0 h1:qPMePEczgbkiQsqCsRfuHRqvDUO+zmAInDaD5ptXlq0= github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= -github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210409134258-03c10cc3d4ea h1:pXD01QLdHmn4Ij82g1vksWbZXwSH6il7Svrm/rdUk18= -github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210409134258-03c10cc3d4ea/go.mod h1:kiPs9g148eLShc2TYagUAyKDnD+dH9U+CQKsXzlY9xo= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -41,10 +39,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed h1:uPxWBzB3+mlnjy9W58qY1j/cjyFjutgw/Vhan2zLy/A= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/v2/go.mod b/v2/go.mod index ae26313..457ad49 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -9,4 +9,3 @@ require ( github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea go.etcd.io/bbolt v1.3.5 ) - diff --git a/v2/go.sum b/v2/go.sum index 9339d32..ac5005f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -7,6 +7,7 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -17,6 +18,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -25,8 +27,10 @@ github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7H github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU= github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -34,10 +38,12 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7q github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From f3754d9fc838f5854610f85de751f4f254c748fe Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Wed, 21 Apr 2021 14:12:05 -0500 Subject: [PATCH 6/7] add v1, source to src, remove sys --- go.mod | 3 +-- go.sum | 4 +--- v2/bolt_store.go | 14 +++++++------- v2/bolt_store_test.go | 23 +++++++++++------------ 4 files changed, 20 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index a1e1d62..65f1577 100644 --- a/go.mod +++ b/go.mod @@ -6,5 +6,4 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/hashicorp/go-msgpack v0.5.5 github.com/hashicorp/raft v1.1.0 - golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 // indirect -) + ) diff --git a/go.sum b/go.sum index 02d1658..ca41d0a 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,4 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= \ No newline at end of file diff --git a/v2/bolt_store.go b/v2/bolt_store.go index b935ca0..8bdbbdb 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -6,7 +6,7 @@ import ( "os" "time" - boltdbbolt "github.com/boltdb/bolt" + v1 "github.com/boltdb/bolt" "github.com/hashicorp/raft" "go.etcd.io/bbolt" ) @@ -271,7 +271,7 @@ func (b *BoltStore) Sync() error { return b.conn.Sync() } -// MigratetoV2 reads in the source file path of a BoltDB file +// MigratetoV2 reads in the source file path of a BoltDB file // and outputs all the data migrated to a Bbolt destination file func MigratetoV2(source, destination string) (*BoltStore, error) { _, err := os.Stat(destination) @@ -279,7 +279,7 @@ func MigratetoV2(source, destination string) (*BoltStore, error) { return nil, fmt.Errorf("file exists in destination %v", destination) } - sourceDb, err := boltdbbolt.Open(source, dbFileMode, &boltdbbolt.Options{ + srcDb, err := v1.Open(source, dbFileMode, &v1.Options{ ReadOnly: true, Timeout: 1 * time.Minute, }) @@ -288,11 +288,11 @@ func MigratetoV2(source, destination string) (*BoltStore, error) { } //Start a connection to the source - sourcetx, err := sourceDb.Begin(false) + srctx, err := srcDb.Begin(false) if err != nil { return nil, fmt.Errorf("failed connecting to source database: %v", err) } - defer sourcetx.Rollback() + defer srctx.Rollback() //Create the destination destDb, err := New(Options{Path: destination}) @@ -312,9 +312,9 @@ func MigratetoV2(source, destination string) (*BoltStore, error) { //Loop over both old buckets and set them in the new buckets := [][]byte{dbConf, dbLogs} for _, b := range buckets { - sourceB := sourcetx.Bucket(b) + srcB := srctx.Bucket(b) destB := desttx.Bucket(b) - err = sourceB.ForEach(func(k, v []byte) error { + err = srcB.ForEach(func(k, v []byte) error { return destB.Put(k, v) }) if err != nil { diff --git a/v2/bolt_store_test.go b/v2/bolt_store_test.go index 617a961..65d37f0 100644 --- a/v2/bolt_store_test.go +++ b/v2/bolt_store_test.go @@ -425,16 +425,15 @@ func TestBoltStore_MigratetoV2(t *testing.T) { } defer os.RemoveAll(dir) - var sourceFile, destFile string - sourceFile = filepath.Join(dir, "/sourcepath") - destFile = filepath.Join(dir, "/destpath") + srcFile := filepath.Join(dir, "/sourcepath") + destFile := filepath.Join(dir, "/destpath") // Successfully creates and returns a store - sourceDb, err := v1.NewBoltStore(sourceFile) + srcDb, err := v1.NewBoltStore(srcFile) if err != nil { t.Fatalf("failed creating source database: %s", err) } - defer sourceDb.Close() + defer srcDb.Close() // Set a mock raft log logs := []*raft.Log{ @@ -444,19 +443,19 @@ func TestBoltStore_MigratetoV2(t *testing.T) { } //Store logs source - if err := sourceDb.StoreLogs(logs); err != nil { + if err := srcDb.StoreLogs(logs); err != nil { t.Fatalf("failed storing logs in source database: %s", err) } - sourceResult := new(raft.Log) - if err := sourceDb.GetLog(2, sourceResult); err != nil { + srcResult := new(raft.Log) + if err := srcDb.GetLog(2, srcResult); err != nil { t.Fatalf("failed getting log from source database: %s", err) } - if err := sourceDb.Close(); err != nil { + if err := srcDb.Close(); err != nil { t.Fatalf("failed closing source database: %s", err) } - destDb, err := MigratetoV2(sourceFile, destFile) + destDb, err := MigratetoV2(srcFile, destFile) if err != nil { t.Fatalf("did not migrate successfully, err %v", err) } @@ -467,8 +466,8 @@ func TestBoltStore_MigratetoV2(t *testing.T) { t.Fatalf("failed getting log from destination database: %s", err) } - if !reflect.DeepEqual(sourceResult, destResult) { - t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", sourceResult, destResult) + if !reflect.DeepEqual(srcResult, destResult) { + t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", srcResult, destResult) } } From c9a6551e50be4a925e69ceb11f92a58498a85fb1 Mon Sep 17 00:00:00 2001 From: Sarah Christoff Date: Wed, 21 Apr 2021 16:01:20 -0500 Subject: [PATCH 7/7] T --- v2/bolt_store.go | 4 ++-- v2/bolt_store_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/bolt_store.go b/v2/bolt_store.go index 8bdbbdb..3024a34 100644 --- a/v2/bolt_store.go +++ b/v2/bolt_store.go @@ -271,9 +271,9 @@ func (b *BoltStore) Sync() error { return b.conn.Sync() } -// MigratetoV2 reads in the source file path of a BoltDB file +// MigrateToV2 reads in the source file path of a BoltDB file // and outputs all the data migrated to a Bbolt destination file -func MigratetoV2(source, destination string) (*BoltStore, error) { +func MigrateToV2(source, destination string) (*BoltStore, error) { _, err := os.Stat(destination) if err == nil { return nil, fmt.Errorf("file exists in destination %v", destination) diff --git a/v2/bolt_store_test.go b/v2/bolt_store_test.go index 65d37f0..7a55de0 100644 --- a/v2/bolt_store_test.go +++ b/v2/bolt_store_test.go @@ -417,7 +417,7 @@ func TestBoltStore_SetUint64_GetUint64(t *testing.T) { } } -func TestBoltStore_MigratetoV2(t *testing.T) { +func TestBoltStore_MigrateToV2(t *testing.T) { dir, err := ioutil.TempDir("", t.Name()) if err != nil { @@ -455,7 +455,7 @@ func TestBoltStore_MigratetoV2(t *testing.T) { t.Fatalf("failed closing source database: %s", err) } - destDb, err := MigratetoV2(srcFile, destFile) + destDb, err := MigrateToV2(srcFile, destFile) if err != nil { t.Fatalf("did not migrate successfully, err %v", err) }