Merge pull request #11 from application-research/add-car-to-bs

chore: re-build the configuration settings

README.md

@@ -30,6 +30,41 @@ if err != nil {
 }
 ```
 
+## Set up a node with your own config
+```
+// initialize a node parameter
+params := NewNodeParams{
+    Ctx:       context.Background(),
+    Datastore: NewInMemoryDatastore(),
+}
+
+// create a new config of your own
+newConfig := &Config{
+    Offline:           true,
+    ReprovideInterval: 0,
+    Libp2pKeyFile:     "mykey",
+    ListenAddrs:       []string{"/ip4/127.0.0.1/tcp/0"},
+    AnnounceAddrs:     nil,
+    DatastoreDir: struct {
+        Directory string
+        Options   leveldb.Options
+    }{},
+    Blockstore:              "",
+    NoBlockstoreCache:       false,
+    NoAnnounceContent:       false,
+    NoLimiter:               false,
+    BitswapConfig:           BitswapConfig{},
+    ConnectionManagerConfig: ConnectionManager{},
+}
+
+// set it
+params.Config = params.ConfigurationBuilder(newConfig)
+myNode, err := NewNode(params)
+if err1 != nil {
+    t.Fatal(err)
+}
+```
+
 ## Add/Pin and Get a file
 ```
 node, err := peer.AddPinFile(context.Background(), bytes.NewReader([]byte("letsrebuildtolearnnewthings!")), nil)
@@ -49,3 +84,9 @@ mainDirNode, err := peer.GetDirectory(context.Background(), node)
 - GetFile function to get a file using a CID
 - GetDirectory function to retrieve an entire directory from a ipld.Node
 - Custom bootstrap nodes
+
+## Examples
+There are a few examples on how to utilize the node which includes
+- how to create a running peer node.
+- how to create a CAR file and add it to the peer node
+- how to add a file / dir to the peer node.

examples/addcartopeer.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	whypfs "github.com/application-research/whypfs-core"
+	"github.com/ipfs/go-cid"
+	format "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-merkledag"
+	"github.com/ipld/go-car"
+	"io/ioutil"
+)
+
+// Creating a new whypfs node, bootstrapping it with the default bootstrap peers, adding a file to the whypfs network, and
+// then retrieving the file from the whypfs network.
+func AddCarToPeerV1() {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	whypfsPeer, err := whypfs.NewNode(whypfs.NewNodeParams{
+		Ctx:       ctx,
+		Datastore: whypfs.NewInMemoryDatastore(),
+	})
+	whypfsPeer.BootstrapPeers(whypfs.DefaultBootstrapPeers())
+
+	baseNode := merkledag.NewRawNode([]byte("letsrebuildtolearnnewthings!"))
+	node1 := &merkledag.ProtoNode{}
+	node2 := &merkledag.ProtoNode{}
+	node3 := &merkledag.ProtoNode{}
+	node4 := &merkledag.ProtoNode{}
+	rootNode := &merkledag.ProtoNode{}
+	node1.AddNodeLink("node1 - yehey", baseNode)
+	node1.SetData([]byte("node1 - yehey"))
+	node2.AddNodeLink("node2  - nice", node1)
+	node2.SetData([]byte("node2  - nice"))
+	node3.AddNodeLink("node3 - wow", node2)
+	node3.SetData([]byte("node3 - wow"))
+	node4.AddNodeLink("node4 - cool", node3)
+
+	// file
+	file, err := ioutil.ReadFile("test_file_for_car1.log")
+	node4.SetData(file)
+
+	rootNode.AddNodeLink("root - alright", node4)
+	rootNode.SetData([]byte("root - alright"))
+	go fmt.Println("Root CID before: ", rootNode.Cid().String())
+
+	assertAddNodes(whypfsPeer.DAGService, rootNode, node4, node3, node2, node1, baseNode)
+
+	buf := new(bytes.Buffer)
+	if err := car.WriteCar(context.Background(), whypfsPeer.DAGService, []cid.Cid{rootNode.Cid()}, buf); err != nil {
+		panic(err)
+	}
+	fmt.Println("CAR file size: ", buf.Len())
+	ch, err := car.LoadCar(context.Background(), whypfsPeer.Blockservice.Blockstore(), buf)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Print(rootNode.Cid().String())
+	for _, c := range ch.Roots {
+		rootCid, err := whypfsPeer.Get(ctx, c)
+		fmt.Println("Root CID after: ", rootCid.String())
+		if err != nil {
+			panic(err)
+		}
+		traverseLinks(ctx, whypfsPeer.DAGService, rootCid)
+	}
+}
+
+func assertAddNodes(ds format.DAGService, nds ...format.Node) {
+	for _, nd := range nds {
+		fmt.Println("Adding node: ", nd.Cid().String())
+		if err := ds.Add(context.Background(), nd); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// function to traverse all links
+func traverseLinks(ctx context.Context, ds format.DAGService, nd format.Node) {
+	for _, link := range nd.Links() {
+		node, err := link.GetNode(ctx, ds)
+		if err != nil {
+			panic(err)
+		}
+		fmt.Println("Node CID: ", node.Cid().String())
+		traverseLinks(ctx, ds, node)
+	}
+}

examples/test_file_for_car1.log

@@ -0,0 +1,19 @@
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file
+this is a test file

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/ipfs/go-merkledag v0.8.0
 	github.com/ipfs/go-metrics-interface v0.0.1
 	github.com/ipfs/go-unixfs v0.4.1
+	github.com/ipld/go-car v0.5.0
 	github.com/libp2p/go-libp2p v0.23.4
 	github.com/libp2p/go-libp2p-kad-dht v0.18.0
 	github.com/multiformats/go-multiaddr v0.7.0

go.sum

@@ -493,6 +493,8 @@ github.com/ipfs/go-unixfs v0.4.1 h1:nmJFKvF+khK03PIWyCxxydD/nkQX315NZDcgvRqMXf0=
 github.com/ipfs/go-unixfs v0.4.1/go.mod h1:2SUDFhUSzrcL408B1qpIkJJ5HznnyTzweViPXUAvkNg=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
+github.com/ipld/go-car v0.5.0 h1:kcCEa3CvYMs0iE5BzD5sV7O2EwMiCIp3uF8tA6APQT8=
+github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE=
 github.com/ipld/go-codec-dagpb v1.3.1 h1:yVNlWRQexCa54ln3MSIiUN++ItH7pdhBFhh0hSgZu1w=
 github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY=
 github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=

whypfs.go

@@ -129,8 +129,9 @@ type BitswapConfig struct {
 	TargetMessageSize          int
 }
 
-func (cfg *Config) setDefaults() {
+func SetConfigDefaults() *Config {
 
+	cfg := &Config{}
 	// optimal settings
 	cfg.Offline = false
 	cfg.ReprovideInterval = defaultReprovideInterval
@@ -147,15 +148,60 @@ func (cfg *Config) setDefaults() {
 	cfg.Libp2pKeyFile = filepath.Join("libp2p.key")
 	cfg.ListenAddrs = []string{"/ip4/0.0.0.0/tcp/0"}
 	cfg.AnnounceAddrs = []string{"/ip4/0.0.0.0/tcp/0"}
+
+	return cfg
+}
+
+func (n NewNodeParams) ConfigurationBuilder(config *Config) *Config {
+
+	// get the default configuration and override it using the config
+	defaultConfig := SetConfigDefaults()
+
+	// check each config if it has some value and override the defaultConfig
+	defaultConfig.NoAnnounceContent = config.NoAnnounceContent
+
+	if config.ConnectionManagerConfig.LowWater > 0 {
+		defaultConfig.ConnectionManagerConfig.LowWater = config.ConnectionManagerConfig.LowWater
+	}
+	if config.ConnectionManagerConfig.HighWater > 0 {
+		defaultConfig.ConnectionManagerConfig.HighWater = config.ConnectionManagerConfig.HighWater
+	}
+
+	if config.BitswapConfig.TargetMessageSize > 0 {
+		defaultConfig.BitswapConfig.TargetMessageSize = config.BitswapConfig.TargetMessageSize
+	}
+	if config.BitswapConfig.MaxOutstandingBytesPerPeer > 0 {
+		defaultConfig.BitswapConfig.TargetMessageSize = config.BitswapConfig.TargetMessageSize
+	}
+
+	if config.AnnounceAddrs != nil {
+		defaultConfig.AnnounceAddrs = config.AnnounceAddrs
+	}
+
+	if config.ListenAddrs != nil {
+		defaultConfig.ListenAddrs = config.ListenAddrs
+	}
+
+	if config.Libp2pKeyFile != "" {
+		defaultConfig.Libp2pKeyFile = config.Libp2pKeyFile
+	}
+
+	if config.DatastoreDir.Directory != "datastore" {
+		defaultConfig.DatastoreDir = config.DatastoreDir
+	}
+
+	defaultConfig.DatastoreDir.Options = config.DatastoreDir.Options
+	return defaultConfig
 }
 
 //	NewNode creates a new WhyPFS node with the given configuration.
-func NewNode(
-	nodeParams NewNodeParams) (*Node, error) {
+func NewNode(nodeParams NewNodeParams) (*Node, error) {
+
 	var err error
+
+	// strictly set defaults
 	if nodeParams.Config == nil {
-		nodeParams.Config = &Config{}
-		nodeParams.Config.setDefaults()
+		nodeParams.Config = SetConfigDefaults()
 	}
 
 	if nodeParams.Repo == "" {
@@ -177,9 +223,9 @@ func NewNode(
 
 	nodeParams.Config.Blockstore = ":flatfs:" + filepath.Join(nodeParams.Repo, "blocks")
 
-	// create the node
 	node := &Node{}
 	node.Config = nodeParams.Config
+	// create the node context
 	node.Ctx = nodeParams.Ctx
 	node.Datastore = nodeParams.Datastore
 	node.Dht = nodeParams.Dht

whypfs_test.go

@@ -367,3 +367,38 @@ func TestAddPinDirectoryAndGetFromAnotherNode(t *testing.T) {
 	assert.GreaterOrEqual(t, len(retrieveNode.Links()), 1)
 
 }
+
+func TestOverrideDefaultConfig(t *testing.T) {
+	params := NewNodeParams{
+		Ctx:       context.Background(),
+		Datastore: NewInMemoryDatastore(),
+	}
+	newConfig := &Config{
+		Offline:           true,
+		ReprovideInterval: 0,
+		Libp2pKeyFile:     "mykey",
+		ListenAddrs:       []string{"/ip4/127.0.0.1/tcp/0"},
+		AnnounceAddrs:     nil,
+		DatastoreDir: struct {
+			Directory string
+			Options   leveldb.Options
+		}{},
+		Blockstore:              "",
+		NoBlockstoreCache:       false,
+		NoAnnounceContent:       false,
+		NoLimiter:               false,
+		BitswapConfig:           BitswapConfig{},
+		ConnectionManagerConfig: ConnectionManager{},
+	}
+	params.Config = params.ConfigurationBuilder(newConfig)
+
+	p1, err1 := NewNode(params)
+	if err1 != nil {
+		t.Fatal(err1)
+	}
+
+	assert.Equal(t, p1.Config.Libp2pKeyFile, "mykey")
+	assert.Equal(t, p1.Config.AnnounceAddrs, []string{"/ip4/0.0.0.0/tcp/0"})
+	assert.Equal(t, p1.Config.ListenAddrs, []string{"/ip4/127.0.0.1/tcp/0"})
+	assert.Equal(t, p1.Config.Offline, params.Config.Offline)
+}