diff --git a/blobstore/store.go b/blobstore/store.go index 679a67be..38778d47 100644 --- a/blobstore/store.go +++ b/blobstore/store.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package blobstore implements the filesystem storage and simpathy/want managment for ssb-blobs. package blobstore import ( @@ -15,31 +16,18 @@ import ( "time" "github.com/pkg/errors" - refs "go.mindeco.de/ssb-refs" - "go.cryptoscope.co/luigi" - "go.cryptoscope.co/ssb" -) -var ( - ErrNoSuchBlob = stderr.New("no such blob") + "go.cryptoscope.co/ssb" + refs "go.mindeco.de/ssb-refs" ) -func parseBlobRef(refStr string) (*refs.BlobRef, error) { - ref, err := refs.ParseRef(refStr) - if err != nil { - return nil, err - } - - br, ok := ref.(*refs.BlobRef) - - if !ok { - return nil, fmt.Errorf("ref is not a %T but a %T", br, ref) - } - - return br, nil -} +// ErrNoSuchBlob is returned if the requested blob isn't available +var ErrNoSuchBlob = stderr.New("ssb: no such blob") +// New creates a new BlobStore, storing it's blobs at the given path. +// This store is functionally equivalent to the javascript implementation and thus can share it's path. +// ie: 'ln -s ~/.ssb/blobs ~/.ssb-go/blobs' works to deduplicate the storage. func New(basePath string) (ssb.BlobStore, error) { err := os.MkdirAll(filepath.Join(basePath, "sha256"), 0700) if err != nil { diff --git a/blobstore/want_options.go b/blobstore/want_options.go index 0054bccd..a7a740d3 100644 --- a/blobstore/want_options.go +++ b/blobstore/want_options.go @@ -7,8 +7,10 @@ import ( "github.com/go-kit/kit/metrics" ) +// WantManagerOption is used to tune different aspects of the WantManager. type WantManagerOption func(*wantManager) error +// WantWithContext supplies a context to cancel its operations. func WantWithContext(ctx context.Context) WantManagerOption { return func(mgr *wantManager) error { mgr.longCtx = ctx @@ -16,8 +18,10 @@ func WantWithContext(ctx context.Context) WantManagerOption { } } +// DefaultMaxSize is 5 megabyte. Blobs that are biggere are not fetched. const DefaultMaxSize = 5 * 1024 * 1024 +// WantWithMaxSize can be used to change DefaultMaxSize func WantWithMaxSize(sz uint) WantManagerOption { return func(mgr *wantManager) error { mgr.maxSize = sz @@ -25,6 +29,7 @@ func WantWithMaxSize(sz uint) WantManagerOption { } } +// WantWithLogger sets up the logger which is used for debug and info output. func WantWithLogger(l log.Logger) WantManagerOption { return func(mgr *wantManager) error { mgr.info = l @@ -32,6 +37,7 @@ func WantWithLogger(l log.Logger) WantManagerOption { } } +// WantWithMetrics setup the metrics counters and gauges to monitor the want manager. func WantWithMetrics(g metrics.Gauge, ctr metrics.Counter) WantManagerOption { return func(mgr *wantManager) error { mgr.gauge = g diff --git a/blobstore/wants.go b/blobstore/wants.go index 256dff3d..35415693 100644 --- a/blobstore/wants.go +++ b/blobstore/wants.go @@ -16,13 +16,15 @@ import ( "github.com/pkg/errors" "go.cryptoscope.co/luigi" "go.cryptoscope.co/muxrpc" - refs "go.mindeco.de/ssb-refs" "go.cryptoscope.co/ssb" + refs "go.mindeco.de/ssb-refs" ) -var ErrBlobBlocked = errors.New("blobstore: unable to receive blob") +// ErrBlobBlocked is returned if the want manager is unable to receive a blob after multiple tries +var ErrBlobBlocked = errors.New("ssb: unable to receive blob correctly") +// NewWantManager returns the configured WantManager, using bs for storage and opts to configure it. func NewWantManager(bs ssb.BlobStore, opts ...WantManagerOption) ssb.WantManager { wmgr := &wantManager{ bs: bs, @@ -413,6 +415,8 @@ func (proc *wantProc) updateWants(ctx context.Context, v interface{}, err error) return proc.out.Pour(ctx, newW) } +// GetWithSize is a muxrpc argument helper. +// It can be used to request a blob named _key_ with a different maximum size than the default. type GetWithSize struct { Key *refs.BlobRef `json:"key"` Max uint `json:"max"` @@ -488,24 +492,21 @@ func (proc *wantProc) Pour(ctx context.Context, v interface{}) error { return errors.Wrap(err, "error responding to wants") } +// WantMsg is an array of _wants_, a blob reference with a distance. type WantMsg []ssb.BlobWant -/* turns a blobwant array into one object ala -{ - ref1:dist1, - ref2:dist2, - ... -} -*/ +// MarshalJSON turns a BlobWant slice into one object. +// for example: { ref1:dist1, ref2:dist2, ... } func (msg WantMsg) MarshalJSON() ([]byte, error) { wantsMap := make(map[*refs.BlobRef]int64, len(msg)) for _, want := range msg { wantsMap[want.Ref] = want.Dist } data, err := json.Marshal(wantsMap) - return data, errors.Wrap(err, "WantMsg: error marshalling map?") + return data, errors.Wrap(err, "WantMsg: error marshalling map") } +// UnmarshalJSON turns an object of {ref:dist, ...} relations into a slice of BlobWants. func (msg *WantMsg) UnmarshalJSON(data []byte) error { var directWants []ssb.BlobWant err := json.Unmarshal(data, &directWants) diff --git a/blobstore/wants_test.go b/blobstore/wants_test.go index 8ed67546..dcde1b79 100644 --- a/blobstore/wants_test.go +++ b/blobstore/wants_test.go @@ -103,14 +103,14 @@ func TestWantManager(t *testing.T) { } for refStr, dist := range tc.localWants { - ref, err := parseBlobRef(refStr) + ref, err := refs.ParseBlobRef(refStr) r.NoError(err, "error parsing ref %q", ref) a.NoError(wmgr.WantWithDist(ref, dist), "error wanting local ref") } for refStr := range tc.localWants { - ref, err := parseBlobRef(refStr) + ref, err := refs.ParseBlobRef(refStr) r.NoError(err, "error parsing ref %q", ref) a.Equal(true, wmgr.Wants(ref), "expected want manager to want ref %q, but it doesn't", ref.Ref()) @@ -118,7 +118,7 @@ func TestWantManager(t *testing.T) { var wmsg WantMsg for refStr, dist := range tc.remoteWants { - ref, err := parseBlobRef(refStr) + ref, err := refs.ParseBlobRef(refStr) r.NoError(err, "error parsing ref %q", ref) wmsg = append(wmsg, ssb.BlobWant{Ref: ref, Dist: dist}) diff --git a/client/client.go b/client/client.go index e4ebfac4..9e058ec4 100644 --- a/client/client.go +++ b/client/client.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package client is a a simple muxrpc interface to common ssb methods, similar to npm:ssb-client package client import ( diff --git a/cmd/go-sbot/main.go b/cmd/go-sbot/main.go index 22812aeb..70c7b114 100644 --- a/cmd/go-sbot/main.go +++ b/cmd/go-sbot/main.go @@ -1,5 +1,8 @@ // SPDX-License-Identifier: MIT +// go-sbot hosts the database and p2p server for replication. +// It supplies various flags to contol options. +// See 'go-sbot -h' for a list and their usage. package main import ( diff --git a/graph/graph.go b/graph/graph.go index 6fca0975..4cca3166 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package graph derives trust/block relations by consuming type:contact message and offers lookup APIs between two feeds. package graph import ( diff --git a/indexes/get.go b/indexes/get.go index a3a09f3e..7a756c6d 100644 --- a/indexes/get.go +++ b/indexes/get.go @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT +// Package indexes contains functions to create indexing for 'get(%ref) -> message'. +// Also contains a utility to open the contact trust graph using the repo and graph packages. package indexes import ( @@ -19,26 +21,28 @@ const FolderNameGet = "get" // OpenGet supplies the get(msgRef) -> rootLogSeq idx func OpenGet(r repo.Interface) (librarian.Index, librarian.SinkIndex, error) { - _, idx, sinkIdx, err := repo.OpenBadgerIndex(r, FolderNameGet, updateFn) + _, idx, sinkIdx, err := repo.OpenBadgerIndex(r, FolderNameGet, createFn) if err != nil { return nil, nil, errors.Wrap(err, "error getting get() index") } return idx, sinkIdx, nil } -func updateFn(db *badger.DB) (librarian.SeqSetterIndex, librarian.SinkIndex) { +func createFn(db *badger.DB) (librarian.SeqSetterIndex, librarian.SinkIndex) { idx := libbadger.NewIndex(db, margaret.BaseSeq(0)) - sink := librarian.NewSinkIndex(func(ctx context.Context, seq margaret.Seq, val interface{}, idx librarian.SetterIndex) error { - msg, ok := val.(refs.Message) - if !ok { - err, ok := val.(error) - if ok && margaret.IsErrNulled(err) { - return nil - } - return errors.Errorf("index/get: unexpected message type: %T", val) - } - err := idx.Set(ctx, librarian.Addr(msg.Key().Hash), seq.Seq()) - return errors.Wrapf(err, "index/get: failed to update message %s (seq: %d)", msg.Key().Ref(), seq.Seq()) - }, idx) + sink := librarian.NewSinkIndex(updateFn, idx) return idx, sink } + +func updateFn(ctx context.Context, seq margaret.Seq, val interface{}, idx librarian.SetterIndex) error { + msg, ok := val.(refs.Message) + if !ok { + err, ok := val.(error) + if ok && margaret.IsErrNulled(err) { + return nil + } + return errors.Errorf("index/get: unexpected message type: %T", val) + } + err := idx.Set(ctx, librarian.Addr(msg.Key().Hash), seq.Seq()) + return errors.Wrapf(err, "index/get: failed to update message %s (seq: %d)", msg.Key().Ref(), seq.Seq()) +} diff --git a/internal/extra25519/convert.go b/internal/extra25519/convert.go index c17d56ff..a6016ba3 100644 --- a/internal/extra25519/convert.go +++ b/internal/extra25519/convert.go @@ -1,4 +1,6 @@ -// blatent grep of github.com/agl/ed25519/extra25519 until we have a better solution +// Package extra25519 is blatent grep of github.com/agl/ed25519/extra25519 until we have a better solution +// +// See https://github.com/cryptoscope/ssb/issues/44 for more. package extra25519 import ( diff --git a/internal/mutil/indirect.go b/internal/mutil/indirect.go index 274c73f9..3812439d 100644 --- a/internal/mutil/indirect.go +++ b/internal/mutil/indirect.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package mutil offers some margaret utilities. package mutil import ( @@ -15,6 +16,8 @@ type indirectLog struct { root, indirect margaret.Log } +// Indirect returns a new Log uses the "indirection Log" to lookup entries in the "root log". +// This helps with the existing database abstraction where indexes just point to entries in the root log by sequence numbers. func Indirect(root, indirect margaret.Log) margaret.Log { il := indirectLog{ root: root, diff --git a/internal/muxmux/new_mux.go b/internal/muxmux/new_mux.go index cc6c70a7..66a2b2ae 100644 --- a/internal/muxmux/new_mux.go +++ b/internal/muxmux/new_mux.go @@ -1,3 +1,7 @@ +// Package muxmux offers an improved muxrpc.HandlerMux (think HTTP router). +// compared to the first draft, this one offers specialed handler functions for the different call types (async, source, sink) +// to reduce boilerplate in handlers. +// TODO: if this shows itselv to be worthwhile, it should be moved to go-muxrpc. package muxmux import ( diff --git a/invite/legacy.go b/invite/legacy.go index 2c7d8774..031614b1 100644 --- a/invite/legacy.go +++ b/invite/legacy.go @@ -1,3 +1,5 @@ +// Package invite contains functions for parsing invite codes and dialing a pub as a guest to redeem a token. +// The muxrpc handlers and storage are found in plugins/legacyinvite. package invite import ( diff --git a/message/drains.go b/message/drains.go index 95479109..a096aa24 100644 --- a/message/drains.go +++ b/message/drains.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package message contains abstract verification and publish helpers. package message import ( diff --git a/message/legacy/verify.go b/message/legacy/verify.go index e0c8223f..85da8009 100644 --- a/message/legacy/verify.go +++ b/message/legacy/verify.go @@ -1,5 +1,9 @@ // SPDX-License-Identifier: MIT +// Package legacy how to encode and verify the current ssb messages. +// You most likely want to use legacy.Verify() in most cases. +// +// See https://spec.scuttlebutt.nz/feed/messages.html for more encoding details. package legacy import ( diff --git a/message/multimsg/multimsg.go b/message/multimsg/multimsg.go index 92ce700b..570cb650 100644 --- a/message/multimsg/multimsg.go +++ b/message/multimsg/multimsg.go @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT +// Package multimsg implements a margaret codec to encode multiple kinds of messages to disk. +// Currently _legacy_ and _gabbygrove_ but should be easily extendable. package multimsg import ( diff --git a/message/requests.go b/message/requests.go index 7aac889a..397433ff 100644 --- a/message/requests.go +++ b/message/requests.go @@ -37,9 +37,7 @@ func NewCreateHistArgsFromMap(argMap map[string]interface{}) (*CreateHistArgs, e qry.AsJSON = b } - case "type": - fallthrough - case "id": + case "type", "id": val, ok := v.(string) if !ok { return nil, errors.Errorf("ssb/message: not string (but %T) for %s", v, k) diff --git a/multilogs/private_readidx.go b/multilogs/private_readidx.go index 58a2398b..89c660fc 100644 --- a/multilogs/private_readidx.go +++ b/multilogs/private_readidx.go @@ -24,7 +24,7 @@ import ( const IndexNamePrivates = "privates" -/* legacy +/* deprecated // not strictly a multilog but allows multiple keys and gives us the good resumption func NewPrivateRead(log kitlog.Logger, kps ...*ssb.KeyPair) *Private { return &Private{ diff --git a/multilogs/userfeeds.go b/multilogs/userfeeds.go index 3c81dc16..52d6a670 100644 --- a/multilogs/userfeeds.go +++ b/multilogs/userfeeds.go @@ -13,7 +13,7 @@ import ( const IndexNameFeeds = "userFeeds" -/* legacy +/* deprecated func OpenUserFeeds(r repo.Interface) (*roaring.MultiLog, librarian.SinkIndex, error) { return repo.OpenFileSystemMultiLog(r, IndexNameFeeds, UserFeedsUpdate) } diff --git a/network/network.go b/network/network.go index e065074c..96eae687 100644 --- a/network/network.go +++ b/network/network.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package network implements utilities for dialing and listening to secret-handshake powered muxrpc connections. package network import ( diff --git a/plugins/blobs/blob.go b/plugins/blobs/blob.go index a0d3a1ea..75176369 100644 --- a/plugins/blobs/blob.go +++ b/plugins/blobs/blob.go @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT +// Package blobs implements the muxrpc handlers for npm:ssb-blobs. +// The storage and want-managment is found in the blobstore package. package blobs import ( diff --git a/plugins/control/handler.go b/plugins/control/handler.go index 66ef4507..cb2e4ab1 100644 --- a/plugins/control/handler.go +++ b/plugins/control/handler.go @@ -1,5 +1,8 @@ // SPDX-License-Identifier: MIT +// Package control offers muxrpc helpers to connect to remote peers. +// +// TODO: this is a naming hack, supplies ctrl.connect which should actually be gossip.connect. package control import ( diff --git a/plugins/friends/handler.go b/plugins/friends/handler.go index 0dc496e6..3120e1dd 100644 --- a/plugins/friends/handler.go +++ b/plugins/friends/handler.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package friends supplies some of npm:ssb-friends, namly isFollowing, isBlocking and hops but not hopStream, onEdge or createLayer. package friends import ( @@ -17,11 +18,7 @@ import ( isFollowing: 'async', isBlocking: 'async', - hops: 'async', - -extra: - - follows: 'source', + hops: 'source', blocks: 'source', */ diff --git a/plugins/get/get.go b/plugins/get/get.go index 8a194716..4acdacf0 100644 --- a/plugins/get/get.go +++ b/plugins/get/get.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package get is just a muxrpc wrapper around sbot.Get package get import ( diff --git a/plugins/gossip/handler.go b/plugins/gossip/handler.go index 8665be52..8c104f48 100644 --- a/plugins/gossip/handler.go +++ b/plugins/gossip/handler.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package gossip implements the createHistoryStream muxrpc call. Legacy (non-EBT) Replication of fetching and verifying the selected feeds is found here. package gossip import ( diff --git a/plugins/legacyinvites/service.go b/plugins/legacyinvites/service.go index 95108f5b..79ba6ea6 100644 --- a/plugins/legacyinvites/service.go +++ b/plugins/legacyinvites/service.go @@ -1,3 +1,4 @@ +// Package legacyinvites supplies the follow-back sub protocol for new users. Translates to npm:ssb-invite. package legacyinvites import ( diff --git a/plugins/partial/plug.go b/plugins/partial/plug.go index bf0b792e..bd081639 100644 --- a/plugins/partial/plug.go +++ b/plugins/partial/plug.go @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT +// Package partial is a helper module for ssb-browser-core, enabling to fetch subsets of feeds. +// See https://github.com/arj03/ssb-partial-replication for more. package partial import ( @@ -30,13 +32,6 @@ func (p plugin) Handler() muxrpc.Handler { return p.h } -// "partialReplication":{ -// getFeed: 'source', -// getFeedReverse: 'source', -// getTangle: 'async', -// getMessagesOfType: 'source' -// } - func New(log logging.Interface, fm *gossip.FeedManager, feeds, bytype, roots *roaring.MultiLog, diff --git a/plugins/peerinvites/idx.go b/plugins/peerinvites/idx.go index 6a60a5f2..4d053849 100644 --- a/plugins/peerinvites/idx.go +++ b/plugins/peerinvites/idx.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package peerinvites spplies the server part of the newer invite system: npm:ssb-peer-invites package peerinvites import ( diff --git a/plugins/private/plug.go b/plugins/private/plug.go index ac40bb0c..eed67ba0 100644 --- a/plugins/private/plug.go +++ b/plugins/private/plug.go @@ -1,5 +1,8 @@ // SPDX-License-Identifier: MIT +// Package private suuplies an about to be deprecated way of accessing private messages. +// This supplies the private.read source from npm:ssb-private. +// In the future relevant queries will have a private:true parameter to unbox readable messages automatically. package private import ( diff --git a/plugins/publish/plug.go b/plugins/publish/plug.go index 5d24521b..53963aa7 100644 --- a/plugins/publish/plug.go +++ b/plugins/publish/plug.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package publish is just a muxrpc wrapper around sbot.PublishLog.Publish. package publish import ( diff --git a/plugins/replicate/upto.go b/plugins/replicate/upto.go index 0fcfa33e..4b5a4604 100644 --- a/plugins/replicate/upto.go +++ b/plugins/replicate/upto.go @@ -1,5 +1,8 @@ // SPDX-License-Identifier: MIT +// Package replicate roughly translates to npm:ssb-replicate and only selects which feeds to block and fetch. +// +// TODO: move ctrl.replicate and ctrl.block here. package replicate import ( diff --git a/repo/repo.go b/repo/repo.go index 5a8a3164..c4f637e8 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package repo contains utility modules to open offset logs and create different kinds of indexes. package repo import ( diff --git a/sbot/options.go b/sbot/options.go index c894cb99..2722b9f4 100644 --- a/sbot/options.go +++ b/sbot/options.go @@ -1,5 +1,7 @@ // SPDX-License-Identifier: MIT +// Package sbot ties together network, repo and plugins like graph and blobs into a large server that offers data-access APIs and background replication. +// It's name dates back to a time where ssb-server was still called scuttlebot, in short: sbot. package sbot import ( diff --git a/tests/interop_test.go b/tests/interop_test.go index d44c5e12..45f26317 100644 --- a/tests/interop_test.go +++ b/tests/interop_test.go @@ -1,5 +1,6 @@ // SPDX-License-Identifier: MIT +// Package tests contains test scenarios and helpers to run interoparability tests against the javascript implementation. package tests import (