Merge pull request #23 from hashicorp/transition

Add helper to go from BoltDB -> Bbolt

go.mod

@@ -6,4 +6,4 @@ require (
 	github.com/boltdb/bolt v1.3.1
 	github.com/hashicorp/go-msgpack v0.5.5
 	github.com/hashicorp/raft v1.1.0
-)
+	)

go.sum

@@ -40,6 +40,4 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed h1:uPxWBzB3+mlnjy9W58qY1j/cjyFjutgw/Vhan2zLy/A=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

v2/bolt_store.go

@@ -2,7 +2,11 @@ package raftboltdb
 
 import (
 	"errors"
+	"fmt"
+	"os"
+	"time"
 
+	v1 "github.com/boltdb/bolt"
 	"github.com/hashicorp/raft"
 	"go.etcd.io/bbolt"
 )
@@ -270,3 +274,67 @@ func (b *BoltStore) GetUint64(key []byte) (uint64, error) {
 func (b *BoltStore) Sync() error {
 	return b.conn.Sync()
 }
+
+// MigrateToV2 reads in the source file path of a BoltDB file
+// and outputs all the data migrated to a Bbolt destination file
+func MigrateToV2(source, destination string) (*BoltStore, error) {
+	_, err := os.Stat(destination)
+	if err == nil {
+		return nil, fmt.Errorf("file exists in destination %v", destination)
+	}
+
+	srcDb, err := v1.Open(source, dbFileMode, &v1.Options{
+		ReadOnly: true,
+		Timeout:  1 * time.Minute,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed opening source database: %v", err)
+	}
+
+	//Start a connection to the source
+	srctx, err := srcDb.Begin(false)
+	if err != nil {
+		return nil, fmt.Errorf("failed connecting to source database: %v", err)
+	}
+	defer srctx.Rollback()
+
+	//Create the destination
+	destDb, err := New(Options{Path: destination})
+	if err != nil {
+		return nil, fmt.Errorf("failed creating destination database: %v", err)
+	}
+	//Start a connection to the new
+	desttx, err := destDb.conn.Begin(true)
+	if err != nil {
+		destDb.Close()
+		os.Remove(destination)
+		return nil, fmt.Errorf("failed connecting to destination database: %v", err)
+	}
+
+	defer desttx.Rollback()
+
+	//Loop over both old buckets and set them in the new
+	buckets := [][]byte{dbConf, dbLogs}
+	for _, b := range buckets {
+		srcB := srctx.Bucket(b)
+		destB := desttx.Bucket(b)
+		err = srcB.ForEach(func(k, v []byte) error {
+			return destB.Put(k, v)
+		})
+		if err != nil {
+			destDb.Close()
+			os.Remove(destination)
+			return nil, fmt.Errorf("failed to copy %v bucket: %v", string(b), err)
+		}
+	}
+
+	//If the commit fails, clean up
+	if err := desttx.Commit(); err != nil {
+		destDb.Close()
+		os.Remove(destination)
+		return nil, fmt.Errorf("failed commiting data to destination: %v", err)
+	}
+
+	return destDb, nil
+
+}

v2/bolt_store_test.go

@@ -4,12 +4,14 @@ import (
 	"bytes"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"reflect"
 	"testing"
 	"time"
 
-	"go.etcd.io/bbolt"
 	"github.com/hashicorp/raft"
+	v1 "github.com/hashicorp/raft-boltdb"
+	"go.etcd.io/bbolt"
 )
 
 func testBoltStore(t testing.TB) *BoltStore {
@@ -414,3 +416,58 @@ func TestBoltStore_SetUint64_GetUint64(t *testing.T) {
 		t.Fatalf("bad: %v", val)
 	}
 }
+
+func TestBoltStore_MigrateToV2(t *testing.T) {
+
+	dir, err := ioutil.TempDir("", t.Name())
+	if err != nil {
+		t.Fatalf("err: %s", err)
+	}
+	defer os.RemoveAll(dir)
+
+	srcFile := filepath.Join(dir, "/sourcepath")
+	destFile := filepath.Join(dir, "/destpath")
+
+	// Successfully creates and returns a store
+	srcDb, err := v1.NewBoltStore(srcFile)
+	if err != nil {
+		t.Fatalf("failed creating source database: %s", err)
+	}
+	defer srcDb.Close()
+
+	// Set a mock raft log
+	logs := []*raft.Log{
+		testRaftLog(1, "log1"),
+		testRaftLog(2, "log2"),
+		testRaftLog(3, "log3"),
+	}
+
+	//Store logs source
+	if err := srcDb.StoreLogs(logs); err != nil {
+		t.Fatalf("failed storing logs in source database: %s", err)
+	}
+	srcResult := new(raft.Log)
+	if err := srcDb.GetLog(2, srcResult); err != nil {
+		t.Fatalf("failed getting log from source database: %s", err)
+	}
+
+	if err := srcDb.Close(); err != nil {
+		t.Fatalf("failed closing source database: %s", err)
+	}
+
+	destDb, err := MigrateToV2(srcFile, destFile)
+	if err != nil {
+		t.Fatalf("did not migrate successfully, err %v", err)
+	}
+	defer destDb.Close()
+
+	destResult := new(raft.Log)
+	if err := destDb.GetLog(2, destResult); err != nil {
+		t.Fatalf("failed getting log from destination database: %s", err)
+	}
+
+	if !reflect.DeepEqual(srcResult, destResult) {
+		t.Errorf("BoltDB log did not equal Bbolt log, Boltdb %v, Bbolt: %v", srcResult, destResult)
+	}
+
+}

v2/go.mod

@@ -3,7 +3,9 @@ module github.com/hashicorp/raft-boltdb/v2
 go 1.12
 
 require (
+	github.com/boltdb/bolt v1.3.1
 	github.com/hashicorp/go-msgpack v0.5.5
 	github.com/hashicorp/raft v1.1.0
+	github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea
 	go.etcd.io/bbolt v1.3.5
 )

v2/go.sum

@@ -2,9 +2,12 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
@@ -15,25 +18,32 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
 github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/raft v1.1.0 h1:qPMePEczgbkiQsqCsRfuHRqvDUO+zmAInDaD5ptXlq0=
 github.com/hashicorp/raft v1.1.0/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM=
+github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea h1:RxcPJuutPRM8PUOyiweMmkuNO+RJyfy2jds2gfvgNmU=
+github.com/hashicorp/raft-boltdb v0.0.0-20210409134258-03c10cc3d4ea/go.mod h1:qRd6nFJYYS6Iqnc/8HcUmko2/2Gw8qTFEmxDLii6W5I=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0=
 golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=