From 021dce1343cf0701d56740d9030f451fbee780f4 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Thu, 31 Aug 2023 11:49:54 -0400 Subject: [PATCH 1/3] all: Update for core v2 --- api/api.go | 17 +- api/api_test.go | 626 +++++++++++++++++++++++++++---- api/client.go | 32 +- api/server.go | 72 +++- go.mod | 3 +- go.sum | 6 +- internal/walletutil/manager.go | 4 +- internal/walletutil/store.go | 250 +++++++------ syncer/syncer.go | 313 +++++++++++++--- wallet/seed.go | 4 +- wallet/wallet.go | 661 ++++++++++++++++++--------------- 11 files changed, 1410 insertions(+), 578 deletions(-) diff --git a/api/api.go b/api/api.go index 0dabb0f..d894469 100644 --- a/api/api.go +++ b/api/api.go @@ -4,7 +4,6 @@ import ( "time" "go.sia.tech/core/types" - "go.sia.tech/walletd/wallet" ) // A GatewayPeer is a currently-connected peer. @@ -19,6 +18,18 @@ type GatewayPeer struct { SyncDuration time.Duration `json:"syncDuration"` } +// TxpoolBroadcastRequest is the request type for /txpool/broadcast. +type TxpoolBroadcastRequest struct { + Transactions []types.Transaction `json:"transactions"` + V2Transactions []types.V2Transaction `json:"v2transactions"` +} + +// TxpoolTransactionsResponse is the response type for /txpool/transactions. +type TxpoolTransactionsResponse struct { + Transactions []types.Transaction `json:"transactions"` + V2Transactions []types.V2Transaction `json:"v2transactions"` +} + // WalletBalanceResponse is the response type for /wallets/:name/balance. type WalletBalanceResponse struct { Siacoins types.Currency `json:"siacoins"` @@ -27,8 +38,8 @@ type WalletBalanceResponse struct { // WalletOutputsResponse is the response type for /wallets/:name/outputs. type WalletOutputsResponse struct { - SiacoinOutputs []wallet.SiacoinElement `json:"siacoinOutputs"` - SiafundOutputs []wallet.SiafundElement `json:"siafundOutputs"` + SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` + SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` } // WalletReserveRequest is the request type for /wallets/:name/reserve. diff --git a/api/api_test.go b/api/api_test.go index 1d3ef0f..546ac03 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -1,10 +1,10 @@ package api_test import ( - "encoding/json" "net" "net/http" "testing" + "time" "go.sia.tech/core/chain" "go.sia.tech/core/consensus" @@ -12,77 +12,35 @@ import ( "go.sia.tech/core/types" "go.sia.tech/jape" "go.sia.tech/walletd/api" + "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/internal/walletutil" "go.sia.tech/walletd/syncer" "go.sia.tech/walletd/wallet" + "lukechampine.com/frand" ) -type mockChainManager struct { - sub chain.Subscriber +func testNetwork() (*consensus.Network, types.Block) { + // use a modified version of Zen + n, genesisBlock := chain.TestnetZen() + n.InitialTarget = types.BlockID{0xFF} + n.HardforkDevAddr.Height = 1 + n.HardforkTax.Height = 1 + n.HardforkStorageProof.Height = 1 + n.HardforkOak.Height = 1 + n.HardforkASIC.Height = 1 + n.HardforkFoundation.Height = 1 + n.HardforkV2.AllowHeight = 5 + n.HardforkV2.RequireHeight = 10 + return n, genesisBlock } -func (mockChainManager) TipState() (cs consensus.State) { return } -func (mockChainManager) RecommendedFee() (fee types.Currency) { return } -func (mockChainManager) PoolTransactions() []types.Transaction { return nil } -func (mockChainManager) AddPoolTransactions([]types.Transaction) error { return nil } -func (mockChainManager) UnconfirmedParents(types.Transaction) []types.Transaction { return nil } -func (mockChainManager) BestIndex(height uint64) (i types.ChainIndex, ok bool) { return i, true } - -func (cm *mockChainManager) AddSubscriber(s chain.Subscriber, index types.ChainIndex) error { - cm.sub = s - return nil -} -func (cm *mockChainManager) RemoveSubscriber(s chain.Subscriber) { - cm.sub = nil -} - -func (cm *mockChainManager) sendTxn(txn types.Transaction) { - created := make([]consensus.SiacoinOutputDiff, len(txn.SiacoinOutputs)) - for i := range created { - created[i] = consensus.SiacoinOutputDiff{ - ID: txn.SiacoinOutputID(i), - Output: txn.SiacoinOutputs[i], - } - } - spent := make([]consensus.SiacoinOutputDiff, len(txn.SiacoinInputs)) - for i := range spent { - spent[i] = consensus.SiacoinOutputDiff{ - ID: txn.SiacoinInputs[i].ParentID, - Output: types.SiacoinOutput{ - Value: types.ZeroCurrency, - Address: txn.SiacoinInputs[i].UnlockConditions.UnlockHash(), - }, - } - } - cm.sub.ProcessChainApplyUpdate(&chain.ApplyUpdate{ - Block: types.Block{ - Timestamp: types.CurrentTimestamp(), - Transactions: []types.Transaction{txn}, - }, - Diff: consensus.BlockDiff{ - Transactions: []consensus.TransactionDiff{{ - CreatedSiacoinOutputs: created, - SpentSiacoinOutputs: spent, - }}, - }, - }, true) -} - -type mockSyncer struct{} - -func (mockSyncer) Addr() string { return "" } -func (mockSyncer) Peers() []*gateway.Peer { return nil } -func (mockSyncer) PeerInfo(string) (i syncer.PeerInfo, ok bool) { return } -func (mockSyncer) Connect(addr string) (*gateway.Peer, error) { return nil, nil } -func (mockSyncer) BroadcastTransactionSet(txns []types.Transaction) {} - -func runServer(wm api.WalletManager) (*api.Client, func()) { +func runServer(cm api.ChainManager, s api.Syncer, wm api.WalletManager) (*api.Client, func()) { l, err := net.Listen("tcp", ":0") if err != nil { panic(err) } go func() { - srv := api.NewServer(mockChainManager{}, mockSyncer{}, wm) + srv := api.NewServer(cm, s, wm) http.Serve(l, jape.BasicAuth("password")(srv)) }() c := api.NewClient("http://"+l.Addr().String(), "password") @@ -90,17 +48,27 @@ func runServer(wm api.WalletManager) (*api.Client, func()) { } func TestWallet(t *testing.T) { - var cm mockChainManager - wm := walletutil.NewEphemeralWalletManager(&cm) - sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) + n, genesisBlock := testNetwork() + giftPrivateKey := types.GeneratePrivateKey() + giftAddress := types.StandardUnlockHash(giftPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0] = types.SiacoinOutput{ + Value: types.Siacoins(1), + Address: giftAddress, + } - c, shutdown := runServer(wm) + // create wallets + dbstore, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tip.State) + wm := walletutil.NewEphemeralWalletManager(cm) + sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) + c, shutdown := runServer(cm, nil, wm) defer shutdown() - - if err := c.AddWallet("primary", json.RawMessage(`{"type":"seed"}`)); err != nil { + if err := c.AddWallet("primary", nil); err != nil { t.Fatal(err) } - wc := c.Wallet("primary") if err := wc.Subscribe(0); err != nil { t.Fatal(err) @@ -143,20 +111,45 @@ func TestWallet(t *testing.T) { t.Fatal("bad address list", addresses) } - // simulate a transaction - cm.sendTxn(types.Transaction{ + // send gift to wallet + giftSCOID := genesisBlock.Transactions[0].SiacoinOutputID(0) + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: giftSCOID, + UnlockConditions: types.StandardUnlockConditions(giftPrivateKey.PublicKey()), + }}, SiacoinOutputs: []types.SiacoinOutput{ {Address: addr, Value: types.Siacoins(1).Div64(2)}, {Address: addr, Value: types.Siacoins(1).Div64(2)}, }, - }) + Signatures: []types.TransactionSignature{{ + ParentID: types.Hash256(giftSCOID), + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }}, + } + sig := giftPrivateKey.SignHash(cm.TipState().WholeSigHash(txn, types.Hash256(giftSCOID), 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + + cs := cm.TipState() + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, + Transactions: []types.Transaction{txn}, + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := cm.AddBlocks([]types.Block{b}); err != nil { + t.Fatal(err) + } // get new balance balance, err = wc.Balance() if err != nil { t.Fatal(err) } else if !balance.Siacoins.Equals(types.Siacoins(1)) { - t.Fatal("balance should be 1 SC") + t.Error("balance should be 1 SC, got", balance.Siacoins) } // transaction should appear in history @@ -164,13 +157,500 @@ func TestWallet(t *testing.T) { if err != nil { t.Fatal(err) } else if len(events) == 0 { - t.Fatal("transaction should appear in history") + t.Error("transaction should appear in history") } outputs, _, err := wc.Outputs() if err != nil { t.Fatal(err) } else if len(outputs) != 2 { - t.Fatal("should have two UTXOs") + t.Error("should have two UTXOs, got", len(outputs)) + } +} + +func TestV2(t *testing.T) { + n, genesisBlock := testNetwork() + // gift primary wallet some coins + primaryPrivateKey := types.GeneratePrivateKey() + primaryAddress := types.StandardUnlockHash(primaryPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = primaryAddress + // secondary wallet starts with nothing + secondaryPrivateKey := types.GeneratePrivateKey() + secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) + + // create wallets + dbstore, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm := chain.NewManager(dbstore, tip.State) + wm := walletutil.NewEphemeralWalletManager(cm) + c, shutdown := runServer(cm, nil, wm) + defer shutdown() + if err := c.AddWallet("primary", nil); err != nil { + t.Fatal(err) + } + primary := c.Wallet("primary") + if err := primary.AddAddress(primaryAddress, nil); err != nil { + t.Fatal(err) + } + if err := primary.Subscribe(0); err != nil { + t.Fatal(err) + } + if err := c.AddWallet("secondary", nil); err != nil { + t.Fatal(err) + } + secondary := c.Wallet("secondary") + if err := secondary.AddAddress(secondaryAddress, nil); err != nil { + t.Fatal(err) + } + if err := secondary.Subscribe(0); err != nil { + t.Fatal(err) + } + + // define some helper functions + addBlock := func(txns []types.Transaction, v2txns []types.V2Transaction) error { + cs := cm.TipState() + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, + Transactions: txns, + } + if v2txns != nil { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + Transactions: v2txns, + } + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + return cm.AddBlocks([]types.Block{b}) + } + checkBalances := func(p, s types.Currency) { + t.Helper() + if primaryBalance, err := primary.Balance(); err != nil { + t.Fatal(err) + } else if !primaryBalance.Siacoins.Equals(p) { + t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) + } + if secondaryBalance, err := secondary.Balance(); err != nil { + t.Fatal(err) + } else if !secondaryBalance.Siacoins.Equals(s) { + t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) + } + } + sendV1 := func() error { + t.Helper() + + // which wallet is sending? + key := primaryPrivateKey + dest := secondaryAddress + pbal, sbal := types.ZeroCurrency, types.ZeroCurrency + sces, _, err := primary.Outputs() + if err != nil { + t.Fatal(err) + } + if len(sces) == 0 { + sces, _, err = secondary.Outputs() + if err != nil { + t.Fatal(err) + } + key = secondaryPrivateKey + dest = primaryAddress + pbal = sces[0].SiacoinOutput.Value + } else { + sbal = sces[0].SiacoinOutput.Value + } + sce := sces[0] + + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: types.SiacoinOutputID(sce.ID), + UnlockConditions: types.StandardUnlockConditions(key.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: dest, + Value: sce.SiacoinOutput.Value, + }}, + Signatures: []types.TransactionSignature{{ + ParentID: sce.ID, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }}, + } + sig := key.SignHash(cm.TipState().WholeSigHash(txn, sce.ID, 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + if err := addBlock([]types.Transaction{txn}, nil); err != nil { + return err + } + checkBalances(pbal, sbal) + return nil + } + sendV2 := func() error { + t.Helper() + + // which wallet is sending? + key := primaryPrivateKey + dest := secondaryAddress + pbal, sbal := types.ZeroCurrency, types.ZeroCurrency + sces, _, err := primary.Outputs() + if err != nil { + t.Fatal(err) + } + if len(sces) == 0 { + sces, _, err = secondary.Outputs() + if err != nil { + t.Fatal(err) + } + key = secondaryPrivateKey + dest = primaryAddress + pbal = sces[0].SiacoinOutput.Value + } else { + sbal = sces[0].SiacoinOutput.Value + } + sce := sces[0] + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(key.PublicKey()))}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: dest, + Value: sce.SiacoinOutput.Value, + }}, + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cm.TipState().InputSigHash(txn))} + if err := addBlock(nil, []types.V2Transaction{txn}); err != nil { + return err + } + checkBalances(pbal, sbal) + return nil + } + + // attempt to send primary->secondary with a v2 txn; should fail + if err := sendV2(); err == nil { + t.Fatal("expected v2 txn to be rejected") + } + // use a v1 transaction instead + if err := sendV1(); err != nil { + t.Fatal(err) + } + + // mine past v2 allow height + for cm.Tip().Height <= n.HardforkV2.AllowHeight { + if err := addBlock(nil, nil); err != nil { + t.Fatal(err) + } + } + // now send coins back with a v2 transaction + if err := sendV2(); err != nil { + t.Fatal(err) + } + // v1 transactions should also still work + if err := sendV1(); err != nil { + t.Fatal(err) + } + + // mine past v2 require height + for cm.Tip().Height <= n.HardforkV2.RequireHeight { + if err := addBlock(nil, nil); err != nil { + t.Fatal(err) + } + } + // v1 transactions should no longer work + if err := sendV1(); err == nil { + t.Fatal("expected v1 txn to be rejected") + } + // use a v2 transaction instead + if err := sendV2(); err != nil { + t.Fatal(err) + } +} + +func TestP2P(t *testing.T) { + n, genesisBlock := testNetwork() + // gift primary wallet some coins + primaryPrivateKey := types.GeneratePrivateKey() + primaryAddress := types.StandardUnlockHash(primaryPrivateKey.PublicKey()) + genesisBlock.Transactions[0].SiacoinOutputs[0].Address = primaryAddress + // secondary wallet starts with nothing + secondaryPrivateKey := types.GeneratePrivateKey() + secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) + + // create wallets + dbstore1, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm1 := chain.NewManager(dbstore1, tip.State) + wm1 := walletutil.NewEphemeralWalletManager(cm1) + l1, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer l1.Close() + s1 := syncer.New(l1, cm1, syncerutil.NewEphemeralPeerStore(), gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l1.Addr().String(), + }) + go s1.Run() + c1, shutdown := runServer(cm1, s1, wm1) + defer shutdown() + if err := c1.AddWallet("primary", nil); err != nil { + t.Fatal(err) + } + primary := c1.Wallet("primary") + if err := primary.AddAddress(primaryAddress, nil); err != nil { + t.Fatal(err) + } + if err := primary.Subscribe(0); err != nil { + t.Fatal(err) + } + + dbstore2, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + if err != nil { + t.Fatal(err) + } + cm2 := chain.NewManager(dbstore2, tip.State) + wm2 := walletutil.NewEphemeralWalletManager(cm2) + l2, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + defer l2.Close() + s2 := syncer.New(l2, cm2, syncerutil.NewEphemeralPeerStore(), gateway.Header{ + GenesisID: genesisBlock.ID(), + UniqueID: gateway.GenerateUniqueID(), + NetAddress: l2.Addr().String(), + }) + go s2.Run() + c2, shutdown2 := runServer(cm2, s2, wm2) + defer shutdown2() + if err := c2.AddWallet("secondary", nil); err != nil { + t.Fatal(err) + } + secondary := c2.Wallet("secondary") + if err := secondary.AddAddress(secondaryAddress, nil); err != nil { + t.Fatal(err) + } + if err := secondary.Subscribe(0); err != nil { + t.Fatal(err) + } + + // define some helper functions + addBlock := func() error { + // choose a client at random + c := c1 + if frand.Intn(2) == 0 { + c = c2 + } + + cs, err := c.ConsensusTipState() + if err != nil { + return err + } + + txns, v2txns, err := c.TxpoolTransactions() + if err != nil { + return err + } + b := types.Block{ + ParentID: cs.Index.ID, + Timestamp: types.CurrentTimestamp(), + MinerPayouts: []types.SiacoinOutput{{Address: types.VoidAddress, Value: cs.BlockReward()}}, + Transactions: txns, + } + if len(v2txns) > 0 { + b.V2 = &types.V2BlockData{ + Height: cs.Index.Height + 1, + Transactions: v2txns, + } + b.V2.Commitment = cs.Commitment(cs.TransactionsCommitment(b.Transactions, b.V2Transactions()), b.MinerPayouts[0].Address) + } + for b.ID().CmpWork(cs.ChildTarget) < 0 { + b.Nonce += cs.NonceFactor() + } + if err := c.SyncerBroadcastBlock(b); err != nil { + return err + } + // wait for tips to update + again: + time.Sleep(10 * time.Millisecond) + if tip1, err := c1.ConsensusTip(); err != nil { + return err + } else if tip2, err := c2.ConsensusTip(); err != nil { + return err + } else if tip1 == cs.Index || tip2 == cs.Index { + goto again + } + return nil + } + checkBalances := func(p, s types.Currency) { + t.Helper() + if primaryBalance, err := primary.Balance(); err != nil { + t.Fatal(err) + } else if !primaryBalance.Siacoins.Equals(p) { + t.Fatalf("primary should have balance of %v, got %v", p, primaryBalance.Siacoins) + } + if secondaryBalance, err := secondary.Balance(); err != nil { + t.Fatal(err) + } else if !secondaryBalance.Siacoins.Equals(s) { + t.Fatalf("secondary should have balance of %v, got %v", s, secondaryBalance.Siacoins) + } + } + sendV1 := func() error { + t.Helper() + + // which wallet is sending? + c := c1 + key := primaryPrivateKey + dest := secondaryAddress + pbal, sbal := types.ZeroCurrency, types.ZeroCurrency + sces, _, err := primary.Outputs() + if err != nil { + t.Fatal(err) + } + if len(sces) == 0 { + c = c2 + key = secondaryPrivateKey + dest = primaryAddress + sces, _, err = secondary.Outputs() + if err != nil { + t.Fatal(err) + } + pbal = sces[0].SiacoinOutput.Value + } else { + sbal = sces[0].SiacoinOutput.Value + } + sce := sces[0] + + txn := types.Transaction{ + SiacoinInputs: []types.SiacoinInput{{ + ParentID: types.SiacoinOutputID(sce.ID), + UnlockConditions: types.StandardUnlockConditions(key.PublicKey()), + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: dest, + Value: sce.SiacoinOutput.Value, + }}, + Signatures: []types.TransactionSignature{{ + ParentID: sce.ID, + CoveredFields: types.CoveredFields{WholeTransaction: true}, + }}, + } + cs, err := c.ConsensusTipState() + if err != nil { + return err + } + sig := key.SignHash(cs.WholeSigHash(txn, sce.ID, 0, 0, nil)) + txn.Signatures[0].Signature = sig[:] + if err := c.TxpoolBroadcast([]types.Transaction{txn}, nil); err != nil { + return err + } else if err := addBlock(); err != nil { + return err + } + checkBalances(pbal, sbal) + return nil + } + sendV2 := func() error { + t.Helper() + + // which wallet is sending? + c := c1 + key := primaryPrivateKey + dest := secondaryAddress + pbal, sbal := types.ZeroCurrency, types.ZeroCurrency + sces, _, err := primary.Outputs() + if err != nil { + t.Fatal(err) + } + if len(sces) == 0 { + c = c2 + key = secondaryPrivateKey + dest = primaryAddress + sces, _, err = secondary.Outputs() + if err != nil { + t.Fatal(err) + } + pbal = sces[0].SiacoinOutput.Value + } else { + sbal = sces[0].SiacoinOutput.Value + } + sce := sces[0] + + txn := types.V2Transaction{ + SiacoinInputs: []types.V2SiacoinInput{{ + Parent: sce, + SatisfiedPolicy: types.SatisfiedPolicy{ + Policy: types.SpendPolicy{Type: types.PolicyTypeUnlockConditions(types.StandardUnlockConditions(key.PublicKey()))}, + }, + }}, + SiacoinOutputs: []types.SiacoinOutput{{ + Address: dest, + Value: sce.SiacoinOutput.Value, + }}, + } + cs, err := c.ConsensusTipState() + if err != nil { + return err + } + txn.SiacoinInputs[0].SatisfiedPolicy.Signatures = []types.Signature{key.SignHash(cs.InputSigHash(txn))} + if err := c.TxpoolBroadcast(nil, []types.V2Transaction{txn}); err != nil { + return err + } else if err := addBlock(); err != nil { + return err + } + checkBalances(pbal, sbal) + return nil + } + + // connect the syncers + if _, err := s1.Connect(s2.Addr()); err != nil { + t.Fatal(err) + } + + // attempt to send primary->secondary with a v2 txn; should fail + if err := sendV2(); err == nil { + t.Fatal("expected v2 txn to be rejected") + } + // use a v1 transaction instead + if err := sendV1(); err != nil { + t.Fatal(err) + } + + // mine past v2 allow height + for cm1.Tip().Height <= n.HardforkV2.AllowHeight { + if err := addBlock(); err != nil { + t.Fatal(err) + } + } + // now send coins back with a v2 transaction + if err := sendV2(); err != nil { + t.Fatal(err) + } + // v1 transactions should also still work + if err := sendV1(); err != nil { + t.Fatal(err) + } + + // mine past v2 require height + for cm1.Tip().Height <= n.HardforkV2.RequireHeight { + if err := addBlock(); err != nil { + t.Fatal(err) + } + } + // v1 transactions should no longer work + if err := sendV1(); err == nil { + t.Fatal("expected v1 txn to be rejected") + } + // use a v2 transaction instead + if err := sendV2(); err != nil { + t.Fatal(err) } } diff --git a/api/client.go b/api/client.go index 8a02c94..973f194 100644 --- a/api/client.go +++ b/api/client.go @@ -14,18 +14,20 @@ import ( // A Client provides methods for interacting with a walletd API server. type Client struct { c jape.Client + n *consensus.Network // for ConsensusTipState } // TxpoolBroadcast broadcasts a set of transaction to the network. -func (c *Client) TxpoolBroadcast(txns []types.Transaction) (err error) { - err = c.c.POST("/txpool/broadcast", txns, nil) +func (c *Client) TxpoolBroadcast(txns []types.Transaction, v2txns []types.V2Transaction) (err error) { + err = c.c.POST("/txpool/broadcast", TxpoolBroadcastRequest{txns, v2txns}, nil) return } // TxpoolTransactions returns all transactions in the transaction pool. -func (c *Client) TxpoolTransactions() (resp []types.Transaction, err error) { +func (c *Client) TxpoolTransactions() (txns []types.Transaction, v2txns []types.V2Transaction, err error) { + var resp TxpoolTransactionsResponse err = c.c.GET("/txpool/transactions", &resp) - return + return resp.Transactions, resp.V2Transactions, err } // TxpoolFee returns the recommended fee (per weight unit) to ensure a high @@ -36,8 +38,9 @@ func (c *Client) TxpoolFee() (resp types.Currency, err error) { } // ConsensusNetwork returns the node's network metadata. -func (c *Client) ConsensusNetwork() (resp consensus.Network, err error) { - err = c.c.GET("/consensus/network", &resp) +func (c *Client) ConsensusNetwork() (resp *consensus.Network, err error) { + resp = new(consensus.Network) + err = c.c.GET("/consensus/network", resp) return } @@ -49,7 +52,14 @@ func (c *Client) ConsensusTip() (resp types.ChainIndex, err error) { // ConsensusTipState returns the current tip state. func (c *Client) ConsensusTipState() (resp consensus.State, err error) { + if c.n == nil { + c.n, err = c.ConsensusNetwork() + if err != nil { + return + } + } err = c.c.GET("/consensus/tipstate", &resp) + resp.Network = c.n return } @@ -65,6 +75,12 @@ func (c *Client) SyncerConnect(addr string) (err error) { return } +// SyncerBroadcastBlock broadcasts a block to all peers. +func (c *Client) SyncerBroadcastBlock(b types.Block) (err error) { + err = c.c.POST("/syncer/broadcast/block", b, nil) + return +} + // Wallets returns the set of tracked wallets. func (c *Client) Wallets() (ws map[string]json.RawMessage, err error) { err = c.c.GET("/wallets", &ws) @@ -141,7 +157,7 @@ func (c *WalletClient) PoolTransactions() (resp []wallet.PoolTransaction, err er } // Outputs returns the set of unspent outputs controlled by the wallet. -func (c *WalletClient) Outputs() (sc []wallet.SiacoinElement, sf []wallet.SiafundElement, err error) { +func (c *WalletClient) Outputs() (sc []types.SiacoinElement, sf []types.SiafundElement, err error) { var resp WalletOutputsResponse err = c.c.GET(fmt.Sprintf("/wallets/%v/outputs", c.name), &resp) return resp.SiacoinOutputs, resp.SiafundOutputs, err @@ -190,7 +206,7 @@ func (c *WalletClient) FundSF(txn types.Transaction, amount uint64, changeAddr, // NewClient returns a client that communicates with a walletd server listening // on the specified address. func NewClient(addr, password string) *Client { - return &Client{jape.Client{ + return &Client{c: jape.Client{ BaseURL: addr, Password: password, }} diff --git a/api/server.go b/api/server.go index 0c4b6e3..d78da6f 100644 --- a/api/server.go +++ b/api/server.go @@ -23,10 +23,12 @@ type ( // A ChainManager manages blockchain and txpool state. ChainManager interface { TipState() consensus.State - + AddBlocks([]types.Block) error RecommendedFee() types.Currency PoolTransactions() []types.Transaction + V2PoolTransactions() []types.V2Transaction AddPoolTransactions(txns []types.Transaction) error + AddV2PoolTransactions(txns []types.V2Transaction) error UnconfirmedParents(txn types.Transaction) []types.Transaction } @@ -36,7 +38,10 @@ type ( Peers() []*gateway.Peer PeerInfo(peer string) (syncer.PeerInfo, bool) Connect(addr string) (*gateway.Peer, error) + BroadcastHeader(bh gateway.BlockHeader) BroadcastTransactionSet(txns []types.Transaction) + BroadcastV2TransactionSet(txns []types.V2Transaction) + BroadcastV2BlockOutline(bo gateway.V2BlockOutline) } // A WalletManager manages wallets, keyed by name. @@ -50,7 +55,7 @@ type ( RemoveAddress(name string, addr types.Address) error Addresses(name string) (map[types.Address]json.RawMessage, error) Events(name string, offset, limit int) ([]wallet.Event, error) - UnspentOutputs(name string) ([]wallet.SiacoinElement, []wallet.SiafundElement, error) + UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) Annotate(name string, pool []types.Transaction) ([]wallet.PoolTransaction, error) } ) @@ -107,8 +112,30 @@ func (s *server) syncerConnectHandler(jc jape.Context) { jc.Check("couldn't connect to peer", err) } +func (s *server) syncerBroadcastBlockHandler(jc jape.Context) { + var b types.Block + if jc.Decode(&b) != nil { + return + } else if jc.Check("block is invalid", s.cm.AddBlocks([]types.Block{b})) != nil { + return + } + if b.V2 == nil { + s.s.BroadcastHeader(gateway.BlockHeader{ + ParentID: b.ParentID, + Nonce: b.Nonce, + Timestamp: b.Timestamp, + MerkleRoot: b.MerkleRoot(), + }) + } else { + s.s.BroadcastV2BlockOutline(gateway.OutlineBlock(b, s.cm.PoolTransactions(), s.cm.V2PoolTransactions())) + } +} + func (s *server) txpoolTransactionsHandler(jc jape.Context) { - jc.Encode(s.cm.PoolTransactions()) + jc.Encode(TxpoolTransactionsResponse{ + Transactions: s.cm.PoolTransactions(), + V2Transactions: s.cm.V2PoolTransactions(), + }) } func (s *server) txpoolFeeHandler(jc jape.Context) { @@ -116,14 +143,22 @@ func (s *server) txpoolFeeHandler(jc jape.Context) { } func (s *server) txpoolBroadcastHandler(jc jape.Context) { - var txnSet []types.Transaction - if jc.Decode(&txnSet) != nil { + var tbr TxpoolBroadcastRequest + if jc.Decode(&tbr) != nil { return } - if jc.Check("invalid transaction set", s.cm.AddPoolTransactions(txnSet)) != nil { - return + if len(tbr.Transactions) != 0 { + if jc.Check("invalid transaction set", s.cm.AddPoolTransactions(tbr.Transactions)) != nil { + return + } + s.s.BroadcastTransactionSet(tbr.Transactions) + } + if len(tbr.V2Transactions) != 0 { + if jc.Check("invalid v2 transaction set", s.cm.AddV2PoolTransactions(tbr.V2Transactions)) != nil { + return + } + s.s.BroadcastV2TransactionSet(tbr.V2Transactions) } - s.s.BroadcastTransactionSet(txnSet) } func (s *server) walletsHandler(jc jape.Context) { @@ -204,10 +239,10 @@ func (s *server) walletsBalanceHandler(jc jape.Context) { var sc types.Currency var sf uint64 for _, sco := range scos { - sc = sc.Add(sco.Value) + sc = sc.Add(sco.SiacoinOutput.Value) } for _, sfo := range sfos { - sf += sfo.Value + sf += sfo.SiafundOutput.Value } jc.Encode(WalletBalanceResponse{ Siacoins: sc, @@ -313,7 +348,7 @@ func (s *server) walletsReleaseHandler(jc jape.Context) { } func (s *server) walletsFundHandler(jc jape.Context) { - fundTxn := func(txn *types.Transaction, amount types.Currency, utxos []wallet.SiacoinElement, changeAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { + fundTxn := func(txn *types.Transaction, amount types.Currency, utxos []types.SiacoinElement, changeAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { s.mu.Lock() defer s.mu.Unlock() if amount.IsZero() { @@ -327,13 +362,13 @@ func (s *server) walletsFundHandler(jc jape.Context) { } frand.Shuffle(len(utxos), reflect.Swapper(utxos)) var outputSum types.Currency - var fundingElements []wallet.SiacoinElement + var fundingElements []types.SiacoinElement for _, sce := range utxos { if s.used[types.Hash256(sce.ID)] || inPool[types.Hash256(sce.ID)] { continue } fundingElements = append(fundingElements, sce) - outputSum = outputSum.Add(sce.Value) + outputSum = outputSum.Add(sce.SiacoinOutput.Value) if outputSum.Cmp(amount) >= 0 { break } @@ -383,7 +418,7 @@ func (s *server) walletsFundHandler(jc jape.Context) { } func (s *server) walletsFundSFHandler(jc jape.Context) { - fundTxn := func(txn *types.Transaction, amount uint64, utxos []wallet.SiafundElement, changeAddr, claimAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { + fundTxn := func(txn *types.Transaction, amount uint64, utxos []types.SiafundElement, changeAddr, claimAddr types.Address, pool []types.Transaction) ([]types.Hash256, error) { s.mu.Lock() defer s.mu.Unlock() if amount == 0 { @@ -397,13 +432,13 @@ func (s *server) walletsFundSFHandler(jc jape.Context) { } frand.Shuffle(len(utxos), reflect.Swapper(utxos)) var outputSum uint64 - var fundingElements []wallet.SiafundElement + var fundingElements []types.SiafundElement for _, sfe := range utxos { if s.used[types.Hash256(sfe.ID)] || inPool[types.Hash256(sfe.ID)] { continue } fundingElements = append(fundingElements, sfe) - outputSum += sfe.Value + outputSum += sfe.SiafundOutput.Value if outputSum >= amount { break } @@ -466,8 +501,9 @@ func NewServer(cm ChainManager, s Syncer, wm WalletManager) http.Handler { "GET /consensus/tip": srv.consensusTipHandler, "GET /consensus/tipstate": srv.consensusTipStateHandler, - "GET /syncer/peers": srv.syncerPeersHandler, - "POST /syncer/connect": srv.syncerConnectHandler, + "GET /syncer/peers": srv.syncerPeersHandler, + "POST /syncer/connect": srv.syncerConnectHandler, + "POST /syncer/broadcast/block": srv.syncerBroadcastBlockHandler, "GET /txpool/transactions": srv.txpoolTransactionsHandler, "GET /txpool/fee": srv.txpoolFeeHandler, diff --git a/go.mod b/go.mod index a9c71fd..211885d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20230719234329-9075f197844e + go.sia.tech/core v0.1.12-0.20230915021325-3ca4ff703dc6 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.9.0 golang.org/x/term v0.6.0 @@ -14,6 +14,7 @@ require ( require ( github.com/aead/chacha20 v0.0.0-20180709150244-8b13a72661da // indirect github.com/julienschmidt/httprouter v1.3.0 // indirect + go.sia.tech/mux v1.2.0 // indirect go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 // indirect golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 // indirect golang.org/x/sys v0.6.0 // indirect diff --git a/go.sum b/go.sum index dc5226d..19b938b 100644 --- a/go.sum +++ b/go.sum @@ -7,10 +7,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20230719234329-9075f197844e h1:B/vnWCN1qyA32cfV61NtesnElFh3UkEFzvMehppdcaE= -go.sia.tech/core v0.1.12-0.20230719234329-9075f197844e/go.mod h1:D17UWSn99SEfQnEaR9G9n6Kz9+BwqMoUgZ6Cl424LsQ= +go.sia.tech/core v0.1.12-0.20230915021325-3ca4ff703dc6 h1:WYQDfzDWBcbr9t+nnatOShOUB4aD2XSu/UGz28XdCwU= +go.sia.tech/core v0.1.12-0.20230915021325-3ca4ff703dc6/go.mod h1:D17UWSn99SEfQnEaR9G9n6Kz9+BwqMoUgZ6Cl424LsQ= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= +go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= +go.sia.tech/mux v1.2.0/go.mod h1:Yyo6wZelOYTyvrHmJZ6aQfRoer3o4xyKQ4NmQLJrBSo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89 h1:wB/JRFeTEs6gviB6k7QARY7Goh54ufkADsdBdn0ZhRo= go.sia.tech/web v0.0.0-20230628194305-c6e1696bad89/go.mod h1:RKODSdOmR3VtObPAcGwQqm4qnqntDVFylbvOBbWYYBU= go.sia.tech/web/walletd v0.9.0 h1:k8nmfOywg/2Z/ggN+Dg+hl+4P1nWITuGdKWJnZkyY5Y= diff --git a/internal/walletutil/manager.go b/internal/walletutil/manager.go index fc5d699..77b8d1f 100644 --- a/internal/walletutil/manager.go +++ b/internal/walletutil/manager.go @@ -120,7 +120,7 @@ func (wm *EphemeralWalletManager) Annotate(name string, txns []types.Transaction } // UnspentOutputs implements api.WalletManager. -func (wm *EphemeralWalletManager) UnspentOutputs(name string) ([]wallet.SiacoinElement, []wallet.SiafundElement, error) { +func (wm *EphemeralWalletManager) UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) { wm.mu.Lock() defer wm.mu.Unlock() mw, ok := wm.wallets[name] @@ -333,7 +333,7 @@ func (wm *JSONWalletManager) Annotate(name string, txns []types.Transaction) ([] } // UnspentOutputs implements api.WalletManager. -func (wm *JSONWalletManager) UnspentOutputs(name string) ([]wallet.SiacoinElement, []wallet.SiafundElement, error) { +func (wm *JSONWalletManager) UnspentOutputs(name string) ([]types.SiacoinElement, []types.SiafundElement, error) { wm.mu.Lock() defer wm.mu.Unlock() mw, ok := wm.wallets[name] diff --git a/internal/walletutil/store.go b/internal/walletutil/store.go index eb710f4..7e68293 100644 --- a/internal/walletutil/store.go +++ b/internal/walletutil/store.go @@ -15,8 +15,8 @@ import ( type EphemeralStore struct { tip types.ChainIndex addrs map[types.Address]json.RawMessage - scos map[types.SiacoinOutputID]types.SiacoinOutput - sfos map[types.SiafundOutputID]types.SiafundOutput + sces map[types.SiacoinOutputID]types.SiacoinElement + sfes map[types.SiafundOutputID]types.SiafundElement events []wallet.Event mu sync.Mutex } @@ -61,20 +61,14 @@ func (s *EphemeralStore) Annotate(txns []types.Transaction) (ptxns []wallet.Pool } // UnspentOutputs implements api.Wallet. -func (s *EphemeralStore) UnspentOutputs() (scos []wallet.SiacoinElement, sfos []wallet.SiafundElement, err error) { +func (s *EphemeralStore) UnspentOutputs() (sces []types.SiacoinElement, sfes []types.SiafundElement, err error) { s.mu.Lock() defer s.mu.Unlock() - for id, sco := range s.scos { - scos = append(scos, wallet.SiacoinElement{ - ID: id, - SiacoinOutput: sco, - }) + for _, sco := range s.sces { + sces = append(sces, sco) } - for id, sfo := range s.sfos { - sfos = append(sfos, wallet.SiafundElement{ - ID: id, - SiafundOutput: sfo, - }) + for _, sfo := range s.sfes { + sfes = append(sfes, sfo) } return } @@ -108,26 +102,18 @@ func (s *EphemeralStore) RemoveAddress(addr types.Address) error { delete(s.addrs, addr) // filter outputs - for scoid, sco := range s.scos { - if sco.Address == addr { - delete(s.scos, scoid) + for scoid, sce := range s.sces { + if sce.SiacoinOutput.Address == addr { + delete(s.sces, scoid) } } - for sfoid, sfo := range s.sfos { - if sfo.Address == addr { - delete(s.sfos, sfoid) + for sfoid, sfe := range s.sfes { + if sfe.SiafundOutput.Address == addr { + delete(s.sfes, sfoid) } } // filter events - relevantElements := func(sces []wallet.SiacoinElement) bool { - for _, sce := range sces { - if s.ownsAddress(sce.Address) { - return true - } - } - return false - } relevantContract := func(fc types.FileContract) bool { for _, sco := range fc.ValidProofOutputs { if s.ownsAddress(sco.Address) { @@ -141,40 +127,65 @@ func (s *EphemeralStore) RemoveAddress(addr types.Address) error { } return false } + relevantV2Contract := func(fc types.V2FileContract) bool { + return s.ownsAddress(fc.RenterOutput.Address) || s.ownsAddress(fc.HostOutput.Address) + } relevantEvent := func(e wallet.Event) bool { switch e := e.Val.(type) { - case *wallet.EventBlockReward: - return s.ownsAddress(e.Output.Address) - case *wallet.EventFoundationSubsidy: - return s.ownsAddress(e.Output.Address) - case *wallet.EventSiacoinMaturation: - return s.ownsAddress(e.Output.Address) - case *wallet.EventSiacoinTransfer: - return relevantElements(e.Inputs) || relevantElements(e.Outputs) - case *wallet.EventSiafundTransfer: - for _, sci := range e.Inputs { - if s.ownsAddress(sci.Address) { + case *wallet.EventTransaction: + for _, sce := range e.SiacoinInputs { + if s.ownsAddress(sce.SiacoinOutput.Address) { + return true + } + } + for _, sce := range e.SiacoinOutputs { + if s.ownsAddress(sce.SiacoinOutput.Address) { return true } } - for _, sco := range e.Outputs { - if s.ownsAddress(sco.Address) { + for _, sfe := range e.SiafundInputs { + if s.ownsAddress(sfe.SiafundElement.SiafundOutput.Address) || + s.ownsAddress(sfe.ClaimElement.SiacoinOutput.Address) { + return true + } + } + for _, sfe := range e.SiafundOutputs { + if s.ownsAddress(sfe.SiafundOutput.Address) { + return true + } + } + for _, fc := range e.FileContracts { + if relevantContract(fc.FileContract.FileContract) || (fc.Revision != nil && relevantContract(*fc.Revision)) { + return true + } + } + for _, fc := range e.V2FileContracts { + if relevantV2Contract(fc.FileContract.V2FileContract) || (fc.Revision != nil && relevantV2Contract(*fc.Revision)) { + return true + } + if fc.Resolution != nil { + switch r := fc.Resolution.(type) { + case *types.V2FileContractFinalization: + if relevantV2Contract(types.V2FileContract(*r)) { + return true + } + case *types.V2FileContractRenewal: + if relevantV2Contract(r.FinalRevision) || relevantV2Contract(r.InitialRevision) { + return true + } + } + } + } + return false + case *wallet.EventMinerPayout: + return s.ownsAddress(e.SiacoinOutput.SiacoinOutput.Address) + case *wallet.EventMissedFileContract: + for _, sce := range e.MissedOutputs { + if s.ownsAddress(sce.SiacoinOutput.Address) { return true } } return false - case *wallet.EventFileContractFormation: - return relevantContract(e.Contract) - case *wallet.EventFileContractRevision: - return relevantContract(e.OldContract) || relevantContract(e.NewContract) - case *wallet.EventFileContractResolutionValid: - return s.ownsAddress(e.Output.Address) - case *wallet.EventFileContractResolutionMissed: - return s.ownsAddress(e.Output.Address) - case *wallet.EventHostAnnouncement: - return relevantElements(e.Inputs) - case *wallet.EventTransaction: - return true // TODO default: panic(fmt.Sprintf("unhandled event type %T", e)) } @@ -195,36 +206,40 @@ func (s *EphemeralStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, _ bool) s.mu.Lock() defer s.mu.Unlock() - events := wallet.DiffEvents(cau.Block, cau.Diff, cau.State.Index, s.ownsAddress) + events := wallet.AppliedEvents(cau.State, cau.Block, cau, s.ownsAddress) s.events = append(s.events, events...) - for _, tdiff := range cau.Diff.Transactions { - for _, scod := range tdiff.SpentSiacoinOutputs { - if s.ownsAddress(scod.Output.Address) { - delete(s.scos, scod.ID) - } - } - for _, scod := range tdiff.CreatedSiacoinOutputs { - if s.ownsAddress(scod.Output.Address) { - s.scos[scod.ID] = scod.Output - } - } - for _, sfod := range tdiff.SpentSiafundOutputs { - if s.ownsAddress(sfod.Output.Address) { - delete(s.sfos, sfod.ID) + // update proofs + for id, sce := range s.sces { + cau.UpdateElementProof(&sce.StateElement) + s.sces[id] = sce + } + for id, sfe := range s.sfes { + cau.UpdateElementProof(&sfe.StateElement) + s.sfes[id] = sfe + } + + // add/remove outputs + cau.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { + if s.ownsAddress(sce.SiacoinOutput.Address) { + if spent { + delete(s.sces, types.SiacoinOutputID(sce.ID)) + } else { + sce.MerkleProof = append([]types.Hash256(nil), sce.MerkleProof...) + s.sces[types.SiacoinOutputID(sce.ID)] = sce } } - for _, sfod := range tdiff.CreatedSiafundOutputs { - if s.ownsAddress(sfod.Output.Address) { - s.sfos[sfod.ID] = sfod.Output + }) + cau.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { + if s.ownsAddress(sfe.SiafundOutput.Address) { + if spent { + delete(s.sfes, types.SiafundOutputID(sfe.ID)) + } else { + sfe.MerkleProof = append([]types.Hash256(nil), sfe.MerkleProof...) + s.sfes[types.SiafundOutputID(sfe.ID)] = sfe } } - } - for _, scod := range cau.Diff.MaturedSiacoinOutputs { - if s.ownsAddress(scod.Output.Address) { - s.scos[scod.ID] = scod.Output - } - } + }) s.tip = cau.State.Index return nil @@ -235,38 +250,39 @@ func (s *EphemeralStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error s.mu.Lock() defer s.mu.Unlock() - // TODO: kinda wasteful - events := wallet.DiffEvents(cru.Block, cru.Diff, cru.State.Index, s.ownsAddress) - s.events = s.events[:len(s.events)-len(events)] + // terribly inefficient, but not a big deal because reverts are infrequent + numEvents := len(wallet.AppliedEvents(cru.State, cru.Block, cru, s.ownsAddress)) + s.events = s.events[:len(s.events)-numEvents] - for _, tdiff := range cru.Diff.Transactions { - for _, scod := range tdiff.SpentSiacoinOutputs { - if s.ownsAddress(scod.Output.Address) { - s.scos[scod.ID] = scod.Output - } - } - for _, scod := range tdiff.CreatedSiacoinOutputs { - if s.ownsAddress(scod.Output.Address) { - delete(s.scos, scod.ID) - } - } - for _, sfod := range tdiff.SpentSiafundOutputs { - if s.ownsAddress(sfod.Output.Address) { - s.sfos[sfod.ID] = sfod.Output + for id, sce := range s.sces { + cru.UpdateElementProof(&sce.StateElement) + s.sces[id] = sce + } + for id, sfe := range s.sfes { + cru.UpdateElementProof(&sfe.StateElement) + s.sfes[id] = sfe + } + + cru.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { + if s.ownsAddress(sce.SiacoinOutput.Address) { + if !spent { + delete(s.sces, types.SiacoinOutputID(sce.ID)) + } else { + sce.MerkleProof = append([]types.Hash256(nil), sce.MerkleProof...) + s.sces[types.SiacoinOutputID(sce.ID)] = sce } } - for _, sfod := range tdiff.CreatedSiafundOutputs { - if s.ownsAddress(sfod.Output.Address) { - delete(s.sfos, sfod.ID) + }) + cru.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { + if s.ownsAddress(sfe.SiafundOutput.Address) { + if !spent { + delete(s.sfes, types.SiafundOutputID(sfe.ID)) + } else { + sfe.MerkleProof = append([]types.Hash256(nil), sfe.MerkleProof...) + s.sfes[types.SiafundOutputID(sfe.ID)] = sfe } } - } - for _, scod := range cru.Diff.MaturedSiacoinOutputs { - if s.ownsAddress(scod.Output.Address) { - delete(s.scos, scod.ID) - } - } - + }) s.tip = cru.State.Index return nil } @@ -275,8 +291,8 @@ func (s *EphemeralStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error func NewEphemeralStore() *EphemeralStore { return &EphemeralStore{ addrs: make(map[types.Address]json.RawMessage), - scos: make(map[types.SiacoinOutputID]types.SiacoinOutput), - sfos: make(map[types.SiafundOutputID]types.SiafundOutput), + sces: make(map[types.SiacoinOutputID]types.SiacoinElement), + sfes: make(map[types.SiafundOutputID]types.SiafundElement), } } @@ -287,20 +303,20 @@ type JSONStore struct { } type persistData struct { - Tip types.ChainIndex - Addresses map[types.Address]json.RawMessage - SiacoinOutputs map[types.SiacoinOutputID]types.SiacoinOutput - SiafundOutputs map[types.SiafundOutputID]types.SiafundOutput - Events []wallet.Event + Tip types.ChainIndex + Addresses map[types.Address]json.RawMessage + SiacoinElements map[types.SiacoinOutputID]types.SiacoinElement + SiafundElements map[types.SiafundOutputID]types.SiafundElement + Events []wallet.Event } func (s *JSONStore) save() error { js, err := json.MarshalIndent(persistData{ - Tip: s.tip, - Addresses: s.addrs, - SiacoinOutputs: s.scos, - SiafundOutputs: s.sfos, - Events: s.events, + Tip: s.tip, + Addresses: s.addrs, + SiacoinElements: s.sces, + SiafundElements: s.sfes, + Events: s.events, }, "", " ") if err != nil { return err @@ -337,8 +353,8 @@ func (s *JSONStore) load() error { } s.tip = p.Tip s.addrs = p.Addresses - s.scos = p.SiacoinOutputs - s.sfos = p.SiafundOutputs + s.sces = p.SiacoinElements + s.sfes = p.SiafundElements s.events = p.Events return nil } diff --git a/syncer/syncer.go b/syncer/syncer.go index 4c51b04..f3d4c14 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -18,14 +18,18 @@ import ( // A ChainManager manages blockchain state. type ChainManager interface { History() ([32]types.BlockID, error) - BlocksForHistory(blocks []types.Block, history []types.BlockID) ([]types.Block, error) + BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) Block(id types.BlockID) (types.Block, bool) + SyncCheckpoint(index types.ChainIndex) (types.Block, consensus.State, bool) AddBlocks(blocks []types.Block) error Tip() types.ChainIndex TipState() consensus.State PoolTransaction(txid types.TransactionID) (types.Transaction, bool) AddPoolTransactions(txns []types.Transaction) error + V2PoolTransaction(txid types.TransactionID) (types.V2Transaction, bool) + AddV2PoolTransactions(txns []types.V2Transaction) error + TransactionsForPartialBlock(missing []types.Hash256) ([]types.Transaction, []types.V2Transaction) } // PeerInfo contains metadata about a peer. @@ -65,8 +69,12 @@ type config struct { ConnectTimeout time.Duration ShareNodesTimeout time.Duration SendBlockTimeout time.Duration + SendTransactionsTimeout time.Duration RelayHeaderTimeout time.Duration + RelayBlockOutlineTimeout time.Duration RelayTransactionSetTimeout time.Duration + SendBlocksTimeout time.Duration + MaxSendBlocks uint64 PeerDiscoveryInterval time.Duration SyncInterval time.Duration Logger *log.Logger @@ -111,12 +119,36 @@ func WithSendBlockTimeout(d time.Duration) Option { return func(c *config) { c.SendBlockTimeout = d } } -// WithRelayHeaderTimeout sets the timeout for the RelayHeader RPC. The default -// is 5 seconds. +// WithSendBlocksTimeout sets the timeout for the SendBlocks RPC. The default is +// 120 seconds. +func WithSendBlocksTimeout(d time.Duration) Option { + return func(c *config) { c.SendBlocksTimeout = d } +} + +// WithMaxSendBlocks sets the maximum number of blocks requested per SendBlocks +// RPC. The default is 10. +func WithMaxSendBlocks(n uint64) Option { + return func(c *config) { c.MaxSendBlocks = n } +} + +// WithSendTransactionsTimeout sets the timeout for the SendTransactions RPC. +// The default is 60 seconds. +func WithSendTransactionsTimeout(d time.Duration) Option { + return func(c *config) { c.SendTransactionsTimeout = d } +} + +// WithRelayHeaderTimeout sets the timeout for the RelayHeader and RelayV2Header +// RPCs. The default is 5 seconds. func WithRelayHeaderTimeout(d time.Duration) Option { return func(c *config) { c.RelayHeaderTimeout = d } } +// WithRelayBlockOutlineTimeout sets the timeout for the RelayV2BlockOutline +// RPC. The default is 60 seconds. +func WithRelayBlockOutlineTimeout(d time.Duration) Option { + return func(c *config) { c.RelayBlockOutlineTimeout = d } +} + // WithRelayTransactionSetTimeout sets the timeout for the RelayTransactionSet // RPC. The default is 60 seconds. func WithRelayTransactionSetTimeout(d time.Duration) Option { @@ -157,8 +189,7 @@ type Syncer struct { } type rpcHandler struct { - s *Syncer - blockBuf []types.Block + s *Syncer } func (h *rpcHandler) PeersForShare() (peers []string) { @@ -178,12 +209,42 @@ func (h *rpcHandler) Block(id types.BlockID) (types.Block, error) { return b, nil } -func (h *rpcHandler) BlocksForHistory(history [32]types.BlockID) ([]types.Block, bool, error) { - blocks, err := h.s.cm.BlocksForHistory(h.blockBuf, history[:]) - return blocks, len(blocks) == len(h.blockBuf), err +func (h *rpcHandler) BlocksForHistory(history []types.BlockID, max uint64) ([]types.Block, uint64, error) { + return h.s.cm.BlocksForHistory(history, max) +} + +func (h *rpcHandler) Transactions(index types.ChainIndex, txnHashes []types.Hash256) (txns []types.Transaction, v2txns []types.V2Transaction, _ error) { + if b, ok := h.s.cm.Block(index.ID); ok { + // get txns from block + want := make(map[types.Hash256]bool) + for _, h := range txnHashes { + want[h] = true + } + for _, txn := range b.Transactions { + if want[txn.FullHash()] { + txns = append(txns, txn) + } + } + for _, txn := range b.V2Transactions() { + if want[txn.FullHash()] { + v2txns = append(v2txns, txn) + } + } + return + } + txns, v2txns = h.s.cm.TransactionsForPartialBlock(txnHashes) + return +} + +func (h *rpcHandler) Checkpoint(index types.ChainIndex) (types.Block, consensus.State, error) { + b, cs, ok := h.s.cm.SyncCheckpoint(index) + if !ok { + return types.Block{}, consensus.State{}, errors.New("checkpoint not found") + } + return b, cs, nil } -func (h *rpcHandler) RelayHeader(bh types.BlockHeader, origin *gateway.Peer) { +func (h *rpcHandler) RelayHeader(bh gateway.BlockHeader, origin *gateway.Peer) { if _, ok := h.s.cm.Block(bh.ID()); ok { return // already seen } else if _, ok := h.s.cm.Block(bh.ParentID); !ok { @@ -200,8 +261,8 @@ func (h *rpcHandler) RelayHeader(bh types.BlockHeader, origin *gateway.Peer) { h.s.synced[origin.Addr] = false h.s.mu.Unlock() return - } else if err := consensus.ValidateHeader(cs, bh); err != nil { - h.s.ban(origin, err) // inexcusable + } else if bh.ID().CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent header with insufficient work")) return } @@ -239,6 +300,123 @@ add: h.s.relayTransactionSet(txns, origin) // non-blocking } +func (h *rpcHandler) RelayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { + if _, ok := h.s.cm.Block(bh.Parent.ID); !ok { + h.s.log.Printf("peer %v relayed a v2 header with unknown parent (%v); triggering a resync", origin, bh.Parent.ID) + h.s.mu.Lock() + h.s.synced[origin.Addr] = false + h.s.mu.Unlock() + return + } + cs := h.s.cm.TipState() + bid := bh.ID(cs) + if _, ok := h.s.cm.Block(bid); ok { + // already seen + return + } else if bh.Parent.ID != cs.Index.ID { + // block extends a sidechain, which peer (if honest) believes to be the + // heaviest chain + h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) + h.s.mu.Lock() + h.s.synced[origin.Addr] = false + h.s.mu.Unlock() + return + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent header with insufficient work")) + return + } + + // header is sufficiently valid; relay it + // + // NOTE: The purpose of header announcements is to inform the network as + // quickly as possible that a new block has been found. A proper + // BlockOutline should follow soon after, allowing peers to obtain the + // actual block. As such, we take no action here other than relaying. + h.s.relayV2Header(bh, origin) // non-blocking +} + +func (h *rpcHandler) RelayV2BlockOutline(bo gateway.V2BlockOutline, origin *gateway.Peer) { + if _, ok := h.s.cm.Block(bo.ParentID); !ok { + h.s.log.Printf("peer %v relayed a header with unknown parent (%v); triggering a resync", origin, bo.ParentID) + h.s.mu.Lock() + h.s.synced[origin.Addr] = false + h.s.mu.Unlock() + return + } + cs := h.s.cm.TipState() + bid := bo.ID(cs) + if _, ok := h.s.cm.Block(bid); ok { + // already seen + return + } else if bo.ParentID != cs.Index.ID { + // block extends a sidechain, which peer (if honest) believes to be the + // heaviest chain + h.s.log.Printf("peer %v relayed a header that does not attach to our tip; triggering a resync", origin) + h.s.mu.Lock() + h.s.synced[origin.Addr] = false + h.s.mu.Unlock() + return + } else if bid.CmpWork(cs.ChildTarget) < 0 { + h.s.ban(origin, errors.New("peer sent header with insufficient work")) + return + } + + // block has sufficient work and attaches to our tip, but may be missing + // transactions; first, check for them in our txpool; then, if block is + // still incomplete, request remaining transactions from the peer + txns, v2txns := h.s.cm.TransactionsForPartialBlock(bo.Missing()) + b, missing := bo.Complete(cs, txns, v2txns) + if len(missing) > 0 { + index := types.ChainIndex{ID: bid, Height: cs.Index.Height + 1} + txns, v2txns, err := origin.SendTransactions(index, missing, h.s.config.SendTransactionsTimeout) + if err != nil { + // log-worthy, but not ban-worthy + h.s.log.Printf("couldn't retrieve missing transactions of %v after relay from %v: %v", bid, origin, err) + return + } + b, missing = bo.Complete(cs, txns, v2txns) + if len(missing) > 0 { + // inexcusable + h.s.ban(origin, errors.New("peer sent wrong missing transactions for a block it relayed")) + return + } + } + if err := h.s.cm.AddBlocks([]types.Block{b}); err != nil { + h.s.ban(origin, err) + return + } + + // when we forward the block, exclude any txns that were in our txpool, + // since they're probably present in our peers' txpools as well + // + // NOTE: crucially, we do NOT exclude any txns we had to request from the + // sending peer, since other peers probably don't have them either + bo.RemoveTransactions(txns, v2txns) + + h.s.relayV2BlockOutline(bo, origin) // non-blocking +} + +func (h *rpcHandler) RelayV2TransactionSet(txns []types.V2Transaction, origin *gateway.Peer) { + // if we've already seen these transactions, don't relay them again + for _, txn := range txns { + if _, ok := h.s.cm.V2PoolTransaction(txn.ID()); !ok { + goto add + } + } + return + +add: + if err := h.s.cm.AddV2PoolTransactions(txns); err != nil { + // too risky to ban here (txns are probably just outdated), but at least + // log it if we think we're synced + if b, ok := h.s.cm.Block(h.s.cm.Tip().ID); ok && time.Since(b.Timestamp) < 2*h.s.cm.TipState().BlockInterval() { + h.s.log.Printf("received an invalid transaction set from %v: %v", origin, err) + } + return + } + h.s.relayV2TransactionSet(txns, origin) // non-blocking +} + func (s *Syncer) ban(p *gateway.Peer, err error) { p.SetErr(errors.New("banned")) s.pm.Ban(p.ConnAddr, 24*time.Hour, err.Error()) @@ -282,10 +460,7 @@ func (s *Syncer) runPeer(p *gateway.Peer) { s.mu.Unlock() }() - h := &rpcHandler{ - s: s, - blockBuf: make([]types.Block, 10), - } + h := &rpcHandler{s: s} inflight := make(chan struct{}, s.config.MaxInflightRPCs) for { if p.Err() != nil { @@ -310,7 +485,7 @@ func (s *Syncer) runPeer(p *gateway.Peer) { } } -func (s *Syncer) relayHeader(h types.BlockHeader, origin *gateway.Peer) { +func (s *Syncer) relayHeader(h gateway.BlockHeader, origin *gateway.Peer) { s.mu.Lock() defer s.mu.Unlock() for _, p := range s.peers { @@ -332,6 +507,39 @@ func (s *Syncer) relayTransactionSet(txns []types.Transaction, origin *gateway.P } } +func (s *Syncer) relayV2Header(bh gateway.V2BlockHeader, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin || !p.SupportsV2() { + continue + } + go p.RelayV2Header(bh, s.config.RelayHeaderTimeout) + } +} + +func (s *Syncer) relayV2BlockOutline(pb gateway.V2BlockOutline, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin || !p.SupportsV2() { + continue + } + go p.RelayV2BlockOutline(pb, s.config.RelayBlockOutlineTimeout) + } +} + +func (s *Syncer) relayV2TransactionSet(txns []types.V2Transaction, origin *gateway.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p == origin || !p.SupportsV2() { + continue + } + go p.RelayV2TransactionSet(txns, s.config.RelayTransactionSetTimeout) + } +} + func (s *Syncer) acceptLoop() error { allowConnect := func(peer string) error { s.mu.Lock() @@ -359,17 +567,12 @@ func (s *Syncer) acceptLoop() error { } go func() { defer conn.Close() - p, err := func() (*gateway.Peer, error) { - if err := allowConnect(conn.RemoteAddr().String()); err != nil { - return nil, err - } - return gateway.AcceptPeer(conn, s.header) - }() - if err == nil { - s.log.Printf("accepted inbound connection from %v", conn.RemoteAddr()) - s.runPeer(p) - } else { + if err := allowConnect(conn.RemoteAddr().String()); err != nil { + s.log.Printf("rejected inbound connection from %v: %v", conn.RemoteAddr(), err) + } else if p, err := gateway.Accept(conn, s.header); err != nil { s.log.Printf("failed to accept inbound connection from %v: %v", conn.RemoteAddr(), err) + } else { + s.runPeer(p) } }() } @@ -406,8 +609,7 @@ func (s *Syncer) peerLoop(closeChan <-chan struct{}) error { var peers []*gateway.Peer s.mu.Lock() for _, p := range s.peers { - peers = append(peers, p) - if len(peers) >= 3 { + if peers = append(peers, p); len(peers) >= 3 { break } } @@ -465,8 +667,7 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { if s.synced[p.Addr] { continue } - peers = append(peers, p) - if len(peers) >= 3 { + if peers = append(peers, p); len(peers) >= 3 { break } } @@ -497,7 +698,7 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { oldTime := time.Now() lastPrint := time.Now() startTime, startHeight := oldTime, oldTip.Height - err = p.SendBlocks(history, func(blocks []types.Block) error { + addBlocks := func(blocks []types.Block) error { if err := s.cm.AddBlocks(blocks); err != nil { return err } @@ -512,19 +713,30 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { lastPrint = time.Now() } return nil - }) + } + if p.SupportsV2() { + history := history[:] + err = func() error { + for { + blocks, rem, err := p.SendV2Blocks(history, s.config.MaxSendBlocks, s.config.SendBlocksTimeout) + if err != nil { + return err + } else if addBlocks(blocks); err != nil { + return err + } else if rem == 0 { + return nil + } + history = []types.BlockID{blocks[len(blocks)-1].ID()} + } + }() + } else { + err = p.SendBlocks(history, s.config.SendBlocksTimeout, addBlocks) + } totalBlocks := s.cm.Tip().Height - oldTip.Height if err != nil { - s.log.Printf("sync with %v failed after %v blocks: %v", p, totalBlocks, err) - continue - } - - // if we extended our best chain, rebroadcast our new tip - if newTip := s.cm.Tip(); newTip != oldTip { + s.log.Printf("syncing with %v failed after %v blocks: %v", p, totalBlocks, err) + } else if newTip := s.cm.Tip(); newTip != oldTip { s.log.Printf("finished syncing %v blocks with %v, tip now %v", p, totalBlocks, newTip) - if b, ok := s.cm.Block(newTip.ID); ok { - s.relayHeader(b.Header(), p) // non-blocking - } } else { s.log.Printf("finished syncing with %v, tip unchanged", p) } @@ -590,7 +802,7 @@ func (s *Syncer) Connect(addr string) (*gateway.Peer, error) { if err != nil { return nil, err } - p, err := gateway.DialPeer(conn, s.header) + p, err := gateway.Dial(conn, s.header) if err != nil { return nil, err } @@ -604,11 +816,22 @@ func (s *Syncer) Connect(addr string) (*gateway.Peer, error) { } // BroadcastHeader broadcasts a header to all peers. -func (s *Syncer) BroadcastHeader(h types.BlockHeader) { s.relayHeader(h, nil) } +func (s *Syncer) BroadcastHeader(h gateway.BlockHeader) { s.relayHeader(h, nil) } + +// BroadcastV2Header broadcasts a v2 header to all peers. +func (s *Syncer) BroadcastV2Header(h gateway.V2BlockHeader) { s.relayV2Header(h, nil) } + +// BroadcastV2BlockOutline broadcasts a v2 block outline to all peers. +func (s *Syncer) BroadcastV2BlockOutline(b gateway.V2BlockOutline) { s.relayV2BlockOutline(b, nil) } // BroadcastTransactionSet broadcasts a transaction set to all peers. func (s *Syncer) BroadcastTransactionSet(txns []types.Transaction) { s.relayTransactionSet(txns, nil) } +// BroadcastV2TransactionSet broadcasts a v2 transaction set to all peers. +func (s *Syncer) BroadcastV2TransactionSet(txns []types.V2Transaction) { + s.relayV2TransactionSet(txns, nil) +} + // Peers returns the set of currently-connected peers. func (s *Syncer) Peers() []*gateway.Peer { s.mu.Lock() @@ -642,8 +865,12 @@ func New(l net.Listener, cm ChainManager, pm PeerStore, header gateway.Header, o ConnectTimeout: 5 * time.Second, ShareNodesTimeout: 5 * time.Second, SendBlockTimeout: 60 * time.Second, + SendTransactionsTimeout: 60 * time.Second, RelayHeaderTimeout: 5 * time.Second, + RelayBlockOutlineTimeout: 60 * time.Second, RelayTransactionSetTimeout: 60 * time.Second, + SendBlocksTimeout: 120 * time.Second, + MaxSendBlocks: 10, PeerDiscoveryInterval: 5 * time.Second, SyncInterval: 5 * time.Second, Logger: log.New(io.Discard, "", 0), diff --git a/wallet/seed.go b/wallet/seed.go index 9cdab31..4a9046a 100644 --- a/wallet/seed.go +++ b/wallet/seed.go @@ -53,7 +53,7 @@ type SeedAddressVault struct { func (sav *SeedAddressVault) gen(index uint64) { for index > uint64(len(sav.addrs)) { - sav.addrs[sav.seed.PublicKey(uint64(len(sav.addrs))).StandardAddress()] = uint64(len(sav.addrs)) + sav.addrs[types.StandardAddress(sav.seed.PublicKey(uint64(len(sav.addrs))))] = uint64(len(sav.addrs)) } } @@ -75,7 +75,7 @@ func (sav *SeedAddressVault) NewAddress(desc string) (types.Address, json.RawMes defer sav.mu.Unlock() index := uint64(len(sav.addrs)) - sav.lookahead + 1 sav.gen(index + sav.lookahead) - addr := sav.seed.PublicKey(index).StandardAddress() + addr := types.StandardAddress(sav.seed.PublicKey(index)) return addr, json.RawMessage(fmt.Sprintf(`{"desc":"%s","keyIndex":%d}`, desc, index)) } diff --git a/wallet/wallet.go b/wallet/wallet.go index 550d26e..782c345 100644 --- a/wallet/wallet.go +++ b/wallet/wallet.go @@ -9,18 +9,6 @@ import ( "go.sia.tech/core/types" ) -// A SiacoinElement is a siacoin output paired with its ID. -type SiacoinElement struct { - ID types.SiacoinOutputID - types.SiacoinOutput -} - -// A SiafundElement is a siafund output paired with its ID. -type SiafundElement struct { - ID types.SiafundOutputID - types.SiafundOutput -} - // StandardTransactionSignature is the most common form of TransactionSignature. // It covers the entire transaction, references a sole public key, and has no // timelock. @@ -152,22 +140,29 @@ func Annotate(txn types.Transaction, ownsAddress func(types.Address) bool) PoolT // An Event is something interesting that happened on the Sia blockchain. type Event struct { - Timestamp time.Time Index types.ChainIndex + Timestamp time.Time + Relevant []types.Address Val interface{ eventType() string } } +func (*EventTransaction) eventType() string { return "transaction" } +func (*EventMinerPayout) eventType() string { return "miner payout" } +func (*EventMissedFileContract) eventType() string { return "missed file contract" } + // MarshalJSON implements json.Marshaler. func (e Event) MarshalJSON() ([]byte, error) { val, _ := json.Marshal(e.Val) return json.Marshal(struct { - Timestamp time.Time - Index types.ChainIndex - Type string - Val json.RawMessage + Timestamp time.Time `json:"timestamp"` + Index types.ChainIndex `json:"index"` + Relevant []types.Address `json:"relevant"` + Type string `json:"type"` + Val json.RawMessage `json:"val"` }{ Timestamp: e.Timestamp, Index: e.Index, + Relevant: e.Relevant, Type: e.Val.eventType(), Val: val, }) @@ -178,6 +173,7 @@ func (e *Event) UnmarshalJSON(data []byte) error { var s struct { Timestamp time.Time Index types.ChainIndex + Relevant []types.Address Type string Val json.RawMessage } @@ -186,29 +182,14 @@ func (e *Event) UnmarshalJSON(data []byte) error { } e.Timestamp = s.Timestamp e.Index = s.Index + e.Relevant = s.Relevant switch s.Type { - case (*EventBlockReward)(nil).eventType(): - e.Val = new(EventBlockReward) - case (*EventFoundationSubsidy)(nil).eventType(): - e.Val = new(EventFoundationSubsidy) - case (*EventSiacoinMaturation)(nil).eventType(): - e.Val = new(EventSiacoinMaturation) - case (*EventSiacoinTransfer)(nil).eventType(): - e.Val = new(EventSiacoinTransfer) - case (*EventSiafundTransfer)(nil).eventType(): - e.Val = new(EventSiafundTransfer) - case (*EventFileContractFormation)(nil).eventType(): - e.Val = new(EventFileContractFormation) - case (*EventFileContractRevision)(nil).eventType(): - e.Val = new(EventFileContractRevision) - case (*EventFileContractResolutionValid)(nil).eventType(): - e.Val = new(EventFileContractResolutionValid) - case (*EventFileContractResolutionMissed)(nil).eventType(): - e.Val = new(EventFileContractResolutionMissed) - case (*EventHostAnnouncement)(nil).eventType(): - e.Val = new(EventHostAnnouncement) case (*EventTransaction)(nil).eventType(): e.Val = new(EventTransaction) + case (*EventMinerPayout)(nil).eventType(): + e.Val = new(EventMinerPayout) + case (*EventMissedFileContract)(nil).eventType(): + e.Val = new(EventMissedFileContract) } if e.Val == nil { return fmt.Errorf("unknown event type %q", s.Type) @@ -216,356 +197,418 @@ func (e *Event) UnmarshalJSON(data []byte) error { return json.Unmarshal(s.Val, e.Val) } -func (*EventBlockReward) eventType() string { return "block reward" } -func (*EventFoundationSubsidy) eventType() string { return "foundation subsidy" } -func (*EventSiacoinMaturation) eventType() string { return "siacoin maturation" } -func (*EventSiacoinTransfer) eventType() string { return "siacoin transfer" } -func (*EventSiafundTransfer) eventType() string { return "siafund transfer" } -func (*EventFileContractFormation) eventType() string { return "file contract formation" } -func (*EventFileContractRevision) eventType() string { return "file contract revision" } -func (*EventFileContractResolutionValid) eventType() string { - return "file contract resolution (valid)" -} -func (*EventFileContractResolutionMissed) eventType() string { - return "file contract resolution (missed)" -} -func (*EventHostAnnouncement) eventType() string { return "host announcement" } -func (*EventTransaction) eventType() string { return "transaction" } - -// EventBlockReward represents a block reward. -type EventBlockReward struct { - OutputID types.SiacoinOutputID - Output types.SiacoinOutput - MaturityHeight uint64 +// A HostAnnouncement represents a host announcement within an EventTransaction. +type HostAnnouncement struct { + PublicKey types.PublicKey + NetAddress string } -// EventFoundationSubsidy represents a Foundation subsidy. -type EventFoundationSubsidy struct { - OutputID types.SiacoinOutputID - Output types.SiacoinOutput - MaturityHeight uint64 +// A SiafundInput represents a siafund input within an EventTransaction. +type SiafundInput struct { + SiafundElement types.SiafundElement `json:"siafundElement"` + ClaimElement types.SiacoinElement `json:"claimElement"` } -// EventSiacoinMaturation represents the maturation of a siacoin output. -type EventSiacoinMaturation struct { - OutputID types.SiacoinOutputID - Output types.SiacoinOutput - Source consensus.DelayedOutputSource +// A FileContract represents a file contract within an EventTransaction. +type FileContract struct { + FileContract types.FileContractElement `json:"fileContract"` + // only non-nil if transaction revised contract + Revision *types.FileContract `json:"revision,omitempty"` + // only non-nil if transaction resolved contract + ValidOutputs []types.SiacoinElement `json:"validOutputs,omitempty"` } -// EventSiacoinTransfer represents the transfer of siacoins within a -// transaction. -type EventSiacoinTransfer struct { - TransactionID types.TransactionID - Inputs []SiacoinElement - Outputs []SiacoinElement - Fee types.Currency +// A V2FileContract represents a v2 file contract within an EventTransaction. +type V2FileContract struct { + FileContract types.V2FileContractElement `json:"fileContract"` + // only non-nil if transaction revised contract + Revision *types.V2FileContract `json:"revision,omitempty"` + // only non-nil if transaction resolved contract + Resolution types.V2FileContractResolutionType `json:"resolution,omitempty"` + Outputs []types.SiacoinElement `json:"outputs,omitempty"` } -// EventSiafundTransfer represents the transfer of siafunds within a -// transaction. -type EventSiafundTransfer struct { - TransactionID types.TransactionID - Inputs []SiafundElement - Outputs []SiafundElement - ClaimOutputID types.SiacoinOutputID - ClaimOutput types.SiacoinOutput -} - -// EventFileContractFormation represents the formation of a file contract within -// a transaction. -type EventFileContractFormation struct { - TransactionID types.TransactionID - ContractID types.FileContractID - Contract types.FileContract -} - -// EventFileContractRevision represents the revision of a file contract within -// a transaction. -type EventFileContractRevision struct { - TransactionID types.TransactionID - ContractID types.FileContractID - OldContract types.FileContract - NewContract types.FileContract -} - -// EventFileContractResolutionValid represents the valid resolution of a file -// contract within a transaction. -type EventFileContractResolutionValid struct { - TransactionID types.TransactionID - ContractID types.FileContractID - Contract types.FileContract - OutputID types.SiacoinOutputID - Output types.SiacoinOutput - MaturityHeight uint64 +type EventTransaction struct { + ID types.TransactionID `json:"id"` + SiacoinInputs []types.SiacoinElement `json:"siacoinInputs"` + SiacoinOutputs []types.SiacoinElement `json:"siacoinOutputs"` + SiafundInputs []SiafundInput `json:"siafundInputs"` + SiafundOutputs []types.SiafundElement `json:"siafundOutputs"` + FileContracts []FileContract `json:"fileContracts"` + V2FileContracts []V2FileContract `json:"v2FileContracts"` + HostAnnouncements []HostAnnouncement `json:"hostAnnouncements"` + Fee types.Currency `json:"fee"` } -// EventFileContractResolutionMissed represents the expiration of a file -// contract. -type EventFileContractResolutionMissed struct { - Contract types.FileContract - OutputID types.SiacoinOutputID - Output types.SiacoinOutput - MaturityHeight uint64 +type EventMinerPayout struct { + SiacoinOutput types.SiacoinElement `json:"siacoinOutput"` } -// EventHostAnnouncement represents a host announcement within a transaction. -type EventHostAnnouncement struct { - TransactionID types.TransactionID - PublicKey types.PublicKey - NetAddress string - Inputs []SiacoinElement +type EventMissedFileContract struct { + FileContract types.FileContractElement `json:"fileContract"` + MissedOutputs []types.SiacoinElement `json:"missedOutputs"` } -// EventTransaction represents a generic transaction. -type EventTransaction struct { - TransactionID types.TransactionID - Transaction types.Transaction +type ChainUpdate interface { + ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool)) + ForEachSiafundElement(func(sfe types.SiafundElement, spent bool)) + ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool)) + ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType)) } -// DiffEvents extracts a list of events from a block diff. -func DiffEvents(b types.Block, diff consensus.BlockDiff, index types.ChainIndex, relevant func(types.Address) bool) []Event { +// AppliedEvents extracts a list of relevant events from a chain update. +func AppliedEvents(cs consensus.State, b types.Block, cu ChainUpdate, relevant func(types.Address) bool) []Event { var events []Event - addEvent := func(v interface{ eventType() string }) { + addEvent := func(v interface{ eventType() string }, relevant []types.Address) { + // dedup relevant addresses + seen := make(map[types.Address]bool) + unique := relevant[:0] + for _, addr := range relevant { + if !seen[addr] { + relevant = append(relevant, addr) + seen[addr] = true + } + } + events = append(events, Event{ Timestamp: b.Timestamp, - Index: index, + Index: cs.Index, + Relevant: unique, Val: v, }) } - relevantContract := func(fc types.FileContract) bool { + // do a first pass to see if there's anything relevant in the block + relevantContract := func(fc types.FileContract) (addrs []types.Address) { for _, sco := range fc.ValidProofOutputs { if relevant(sco.Address) { - return true + addrs = append(addrs, sco.Address) } } for _, sco := range fc.MissedProofOutputs { if relevant(sco.Address) { - return true + addrs = append(addrs, sco.Address) } } - return false + return + } + relevantV2Contract := func(fc types.V2FileContract) (addrs []types.Address) { + if relevant(fc.RenterOutput.Address) { + addrs = append(addrs, fc.RenterOutput.Address) + } + if relevant(fc.HostOutput.Address) { + addrs = append(addrs, fc.HostOutput.Address) + } + return + } + relevantV2ContractResolution := func(res types.V2FileContractResolutionType) (addrs []types.Address) { + switch r := res.(type) { + case *types.V2FileContractFinalization: + return relevantV2Contract(types.V2FileContract(*r)) + case *types.V2FileContractRenewal: + return append(relevantV2Contract(r.InitialRevision), relevantV2Contract(r.FinalRevision)...) + } + return } - relevantTxn := func(tdiff consensus.TransactionDiff) bool { - for _, scod := range tdiff.CreatedSiacoinOutputs { - if relevant(scod.Output.Address) { - return true + anythingRelevant := func() (ok bool) { + cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { + if ok || relevant(sce.SiacoinOutput.Address) { + ok = true + } + }) + cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { + if ok || relevant(sfe.SiafundOutput.Address) { + ok = true + } + }) + cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + if ok || len(relevantContract(fce.FileContract)) > 0 || (rev != nil && len(relevantContract(rev.FileContract)) > 0) { + ok = true + } + }) + cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + if ok || + len(relevantV2Contract(fce.V2FileContract)) > 0 || + (rev != nil && len(relevantV2Contract(rev.V2FileContract)) > 0) || + (res != nil && len(relevantV2ContractResolution(res)) > 0) { + ok = true + } + }) + return + }() + if !anythingRelevant { + return nil + } + + // collect all elements + sces := make(map[types.SiacoinOutputID]types.SiacoinElement) + sfes := make(map[types.SiafundOutputID]types.SiafundElement) + fces := make(map[types.FileContractID]types.FileContractElement) + v2fces := make(map[types.FileContractID]types.V2FileContractElement) + cu.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { + sce.MerkleProof = nil + sces[types.SiacoinOutputID(sce.ID)] = sce + }) + cu.ForEachSiafundElement(func(sfe types.SiafundElement, spent bool) { + sfe.MerkleProof = nil + sfes[types.SiafundOutputID(sfe.ID)] = sfe + }) + cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + fce.MerkleProof = nil + fces[types.FileContractID(fce.ID)] = fce + }) + cu.ForEachV2FileContractElement(func(fce types.V2FileContractElement, rev *types.V2FileContractElement, res types.V2FileContractResolutionType) { + fce.MerkleProof = nil + v2fces[types.FileContractID(fce.ID)] = fce + }) + + relevantTxn := func(txn types.Transaction) (addrs []types.Address) { + for _, sci := range txn.SiacoinInputs { + if sce := sces[sci.ParentID]; relevant(sce.SiacoinOutput.Address) { + addrs = append(addrs, sce.SiacoinOutput.Address) } } - for _, dscod := range tdiff.ImmatureSiacoinOutputs { - if relevant(dscod.Output.Address) { - return true + for _, sco := range txn.SiacoinOutputs { + if relevant(sco.Address) { + addrs = append(addrs, sco.Address) } } - for _, sfod := range tdiff.CreatedSiafundOutputs { - if relevant(sfod.Output.Address) { - return true + for _, sfi := range txn.SiafundInputs { + if sfe := sfes[sfi.ParentID]; relevant(sfe.SiafundOutput.Address) { + addrs = append(addrs, sfe.SiafundOutput.Address) } } - for _, fcd := range tdiff.CreatedFileContracts { - if relevantContract(fcd.Contract) { - return true + for _, sfo := range txn.SiafundOutputs { + if relevant(sfo.Address) { + addrs = append(addrs, sfo.Address) } } - for _, scod := range tdiff.SpentSiacoinOutputs { - if relevant(scod.Output.Address) { - return true + for _, fc := range txn.FileContracts { + addrs = append(addrs, relevantContract(fc)...) + } + for _, fcr := range txn.FileContractRevisions { + addrs = append(addrs, relevantContract(fcr.FileContract)...) + } + for _, sp := range txn.StorageProofs { + addrs = append(addrs, relevantContract(fces[sp.ParentID].FileContract)...) + } + return + } + + relevantV2Txn := func(txn types.V2Transaction) (addrs []types.Address) { + for _, sci := range txn.SiacoinInputs { + if relevant(sci.Parent.SiacoinOutput.Address) { + addrs = append(addrs, sci.Parent.SiacoinOutput.Address) } } - for _, sfod := range tdiff.SpentSiafundOutputs { - if relevant(sfod.Output.Address) { - return true + for _, sco := range txn.SiacoinOutputs { + if relevant(sco.Address) { + addrs = append(addrs, sco.Address) + } + } + for _, sfi := range txn.SiafundInputs { + if relevant(sfi.Parent.SiafundOutput.Address) { + addrs = append(addrs, sfi.Parent.SiafundOutput.Address) } } - for _, fcd := range tdiff.RevisedFileContracts { - if relevantContract(fcd.NewContract) { - return true + for _, sfo := range txn.SiafundOutputs { + if relevant(sfo.Address) { + addrs = append(addrs, sfo.Address) } } - for _, fcd := range tdiff.ValidFileContracts { - if relevantContract(fcd.Contract) { - return true + for _, fc := range txn.FileContracts { + addrs = append(addrs, relevantV2Contract(fc)...) + } + for _, fcr := range txn.FileContractRevisions { + addrs = append(addrs, relevantV2Contract(fcr.Parent.V2FileContract)...) + addrs = append(addrs, relevantV2Contract(fcr.Revision)...) + } + for _, fcr := range txn.FileContractResolutions { + addrs = append(addrs, relevantV2Contract(fcr.Parent.V2FileContract)...) + switch r := fcr.Resolution.(type) { + case *types.V2FileContractFinalization: + addrs = append(addrs, relevantV2Contract(types.V2FileContract(*r))...) + case *types.V2FileContractRenewal: + addrs = append(addrs, relevantV2Contract(r.InitialRevision)...) + addrs = append(addrs, relevantV2Contract(r.FinalRevision)...) } } - return false + return } - for i, tdiff := range diff.Transactions { - if !relevantTxn(tdiff) { + // handle v1 transactions + for _, txn := range b.Transactions { + relevant := relevantTxn(txn) + if len(relevant) == 0 { continue } - txn := b.Transactions[i] - txid := txn.ID() - oldLen := len(events) - - var scInputs, scOutputs []SiacoinElement - for _, scod := range tdiff.SpentSiacoinOutputs { - if relevant(scod.Output.Address) { - scInputs = append(scInputs, SiacoinElement{ - ID: scod.ID, - SiacoinOutput: scod.Output, - }) - } + + e := &EventTransaction{ + ID: txn.ID(), + SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), + SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), + SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), + SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), } - for _, scod := range tdiff.CreatedSiacoinOutputs { - if relevant(scod.Output.Address) { - scOutputs = append(scOutputs, SiacoinElement{ - ID: scod.ID, - SiacoinOutput: scod.Output, - }) - } + + for i := range txn.SiacoinInputs { + e.SiacoinInputs[i] = sces[txn.SiacoinInputs[i].ParentID] } - var fee types.Currency - for _, c := range txn.MinerFees { - fee = fee.Add(c) - } - if len(scInputs) > 0 || len(scOutputs) > 0 { - addEvent(&EventSiacoinTransfer{ - TransactionID: txid, - Inputs: scInputs, - Outputs: scOutputs, - Fee: fee, - }) - } - - var sfInputs, sfOutputs []SiafundElement - for _, sfod := range tdiff.SpentSiafundOutputs { - if relevant(sfod.Output.Address) { - sfInputs = append(sfInputs, SiafundElement{ - ID: sfod.ID, - SiafundOutput: sfod.Output, - }) - } + for i := range txn.SiacoinOutputs { + e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(i)] } - for _, sfod := range tdiff.CreatedSiafundOutputs { - if relevant(sfod.Output.Address) { - sfOutputs = append(sfOutputs, SiafundElement{ - ID: sfod.ID, - SiafundOutput: sfod.Output, - }) + for i := range txn.SiafundInputs { + e.SiafundInputs[i] = SiafundInput{ + SiafundElement: sfes[txn.SiafundInputs[i].ParentID], + ClaimElement: sces[txn.SiafundClaimOutputID(i)], } } - if len(sfInputs) > 0 || len(sfOutputs) > 0 { - addEvent(&EventSiafundTransfer{ - TransactionID: txid, - Inputs: sfInputs, - Outputs: sfOutputs, - }) + for i := range txn.SiafundOutputs { + e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(i)] } - - for _, fcd := range tdiff.CreatedFileContracts { - if relevantContract(fcd.Contract) { - addEvent(&EventFileContractFormation{ - TransactionID: txid, - ContractID: fcd.ID, - Contract: fcd.Contract, - }) + addContract := func(id types.FileContractID) *FileContract { + for i := range e.FileContracts { + if types.FileContractID(e.FileContracts[i].FileContract.ID) == id { + return &e.FileContracts[i] + } } + e.FileContracts = append(e.FileContracts, FileContract{FileContract: fces[id]}) + return &e.FileContracts[len(e.FileContracts)-1] } - for _, fcrd := range tdiff.RevisedFileContracts { - if relevantContract(fcrd.OldContract) || relevantContract(fcrd.NewContract) { - addEvent(&EventFileContractRevision{ - TransactionID: txid, - ContractID: fcrd.ID, - OldContract: fcrd.OldContract, - NewContract: fcrd.NewContract, - }) + for i := range txn.FileContracts { + addContract(txn.FileContractID(i)) + } + for i := range txn.FileContractRevisions { + fc := addContract(txn.FileContractRevisions[i].ParentID) + rev := txn.FileContractRevisions[i].FileContract + fc.Revision = &rev + } + for i := range txn.StorageProofs { + fc := addContract(txn.StorageProofs[i].ParentID) + fc.ValidOutputs = make([]types.SiacoinElement, len(fc.FileContract.FileContract.ValidProofOutputs)) + for i := range fc.ValidOutputs { + fc.ValidOutputs[i] = sces[types.FileContractID(fc.FileContract.ID).ValidOutputID(i)] } } - for _, fcd := range tdiff.ValidFileContracts { - if relevantContract(fcd.Contract) { - addEvent(&EventFileContractResolutionValid{ - TransactionID: txid, - ContractID: fcd.ID, - Contract: fcd.Contract, + for _, arb := range txn.ArbitraryData { + var prefix types.Specifier + var uk types.UnlockKey + d := types.NewBufDecoder(arb) + prefix.DecodeFrom(d) + netAddress := d.ReadString() + uk.DecodeFrom(d) + if d.Err() == nil && prefix == types.NewSpecifier("HostAnnouncement") && + uk.Algorithm == types.SpecifierEd25519 && len(uk.Key) == len(types.PublicKey{}) { + e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ + PublicKey: *(*types.PublicKey)(uk.Key), + NetAddress: netAddress, }) } } + for i := range txn.MinerFees { + e.Fee = e.Fee.Add(txn.MinerFees[i]) + } - // only report host announcements that we paid for - if len(scInputs) > 0 { - decodeAnnouncement := func(b []byte) (types.PublicKey, string, bool) { - var prefix types.Specifier - var uk types.UnlockKey - d := types.NewBufDecoder(b) - prefix.DecodeFrom(d) - netAddress := d.ReadString() - uk.DecodeFrom(d) - if d.Err() != nil || - prefix != types.NewSpecifier("HostAnnouncement") || - uk.Algorithm != types.SpecifierEd25519 || - len(uk.Key) != len(types.PublicKey{}) { - return types.PublicKey{}, "", false - } - return *(*types.PublicKey)(uk.Key), netAddress, true + addEvent(e, relevant) + } + + // handle v2 transactions + for _, txn := range b.V2Transactions() { + relevant := relevantV2Txn(txn) + if len(relevant) == 0 { + continue + } + + txid := txn.ID() + e := &EventTransaction{ + ID: txid, + SiacoinInputs: make([]types.SiacoinElement, len(txn.SiacoinInputs)), + SiacoinOutputs: make([]types.SiacoinElement, len(txn.SiacoinOutputs)), + SiafundInputs: make([]SiafundInput, len(txn.SiafundInputs)), + SiafundOutputs: make([]types.SiafundElement, len(txn.SiafundOutputs)), + } + for i := range txn.SiacoinInputs { + // NOTE: here (and elsewhere), we fetch the element from our maps, + // rather than using the parent directly, because our copy has its + // Merkle proof nil'd out + e.SiacoinInputs[i] = sces[types.SiacoinOutputID(txn.SiacoinInputs[i].Parent.ID)] + } + for i := range txn.SiacoinOutputs { + e.SiacoinOutputs[i] = sces[txn.SiacoinOutputID(txid, i)] + } + for i := range txn.SiafundInputs { + sfoid := types.SiafundOutputID(txn.SiafundInputs[i].Parent.ID) + e.SiafundInputs[i] = SiafundInput{ + SiafundElement: sfes[sfoid], + ClaimElement: sces[sfoid.ClaimOutputID()], } - for _, arb := range txn.ArbitraryData { - if pubkey, netAddress, ok := decodeAnnouncement(arb); ok { - addEvent(&EventHostAnnouncement{ - TransactionID: txid, - PublicKey: pubkey, - NetAddress: netAddress, - Inputs: scInputs, - }) + } + for i := range txn.SiafundOutputs { + e.SiafundOutputs[i] = sfes[txn.SiafundOutputID(txid, i)] + } + addContract := func(id types.FileContractID) *V2FileContract { + for i := range e.V2FileContracts { + if types.FileContractID(e.V2FileContracts[i].FileContract.ID) == id { + return &e.V2FileContracts[i] } } + e.V2FileContracts = append(e.V2FileContracts, V2FileContract{FileContract: v2fces[id]}) + return &e.V2FileContracts[len(e.V2FileContracts)-1] } - - if len(events) == oldLen { - // transaction is relevant, but doesn't map to any predefined - // event categories - addEvent(&EventTransaction{ - TransactionID: txid, - Transaction: txn, - }) + for i := range txn.FileContracts { + addContract(txn.V2FileContractID(txid, i)) } - } - - for _, dscod := range diff.MaturedSiacoinOutputs { - if relevant(dscod.Output.Address) { - addEvent(&EventSiacoinMaturation{ - OutputID: dscod.ID, - Output: dscod.Output, - Source: dscod.Source, - }) + for _, fcr := range txn.FileContractRevisions { + fc := addContract(types.FileContractID(fcr.Parent.ID)) + fc.Revision = &fcr.Revision } - } - for _, dscod := range diff.ImmatureSiacoinOutputs { - if relevant(dscod.Output.Address) { - switch dscod.Source { - case consensus.OutputSourceMiner: - addEvent(&EventBlockReward{ - OutputID: dscod.ID, - Output: dscod.Output, - MaturityHeight: dscod.MaturityHeight, - }) - case consensus.OutputSourceMissedContract: - addEvent(&EventFileContractResolutionMissed{ - OutputID: dscod.ID, - Output: dscod.Output, - MaturityHeight: dscod.MaturityHeight, - }) - case consensus.OutputSourceFoundation: - addEvent(&EventBlockReward{ - OutputID: dscod.ID, - Output: dscod.Output, - MaturityHeight: dscod.MaturityHeight, + for _, fcr := range txn.FileContractResolutions { + fc := addContract(types.FileContractID(fcr.Parent.ID)) + fc.Resolution = fcr.Resolution + fc.Outputs = []types.SiacoinElement{ + sces[types.FileContractID(fcr.Parent.ID).V2RenterOutputID()], + sces[types.FileContractID(fcr.Parent.ID).V2HostOutputID()], + } + } + for _, a := range txn.Attestations { + if a.Key == "HostAnnouncement" { + e.HostAnnouncements = append(e.HostAnnouncements, HostAnnouncement{ + PublicKey: a.PublicKey, + NetAddress: string(a.Value), }) } } + + e.Fee = txn.MinerFee + addEvent(e, relevant) } - for _, fcd := range diff.MissedFileContracts { - for i, sco := range fcd.Contract.MissedProofOutputs { - if relevant(sco.Address) { - addEvent(&EventFileContractResolutionMissed{ - Contract: fcd.Contract, - OutputID: fcd.ID.MissedOutputID(i), - Output: sco, - MaturityHeight: diff.ImmatureSiacoinOutputs[0].MaturityHeight, - }) + + // handle missed contracts + cu.ForEachFileContractElement(func(fce types.FileContractElement, rev *types.FileContractElement, resolved, valid bool) { + if resolved && !valid { + relevant := relevantContract(fce.FileContract) + if len(relevant) == 0 { + return + } + missedOutputs := make([]types.SiacoinElement, len(fce.FileContract.MissedProofOutputs)) + for i := range missedOutputs { + missedOutputs[i] = sces[types.FileContractID(fce.ID).MissedOutputID(i)] } + addEvent(&EventMissedFileContract{ + FileContract: fce, + MissedOutputs: missedOutputs, + }, relevant) + } + }) + + // handle block rewards + for i := range b.MinerPayouts { + if relevant(b.MinerPayouts[i].Address) { + addEvent(&EventMinerPayout{ + SiacoinOutput: sces[cs.Index.ID.MinerOutputID(i)], + }, []types.Address{b.MinerPayouts[i].Address}) } } + return events } From e1f65768d69fee4c49abe3eb76944cb830763a09 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sat, 21 Oct 2023 15:50:19 -0400 Subject: [PATCH 2/3] syncer: Improve shutdown speed --- syncer/syncer.go | 111 ++++++++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 40 deletions(-) diff --git a/syncer/syncer.go b/syncer/syncer.go index f3d4c14..ec1597b 100644 --- a/syncer/syncer.go +++ b/syncer/syncer.go @@ -1,6 +1,7 @@ package syncer import ( + "context" "errors" "io" "log" @@ -540,26 +541,44 @@ func (s *Syncer) relayV2TransactionSet(txns []types.V2Transaction, origin *gatew } } -func (s *Syncer) acceptLoop() error { - allowConnect := func(peer string) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.pm.Banned(peer) { - return errors.New("banned") - } - var in int - for _, p := range s.peers { - if p.Inbound { - in++ - } +func (s *Syncer) allowConnect(peer string, inbound bool) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.l == nil { + return errors.New("syncer is shutting down") + } + if s.pm.Banned(peer) { + return errors.New("banned") + } + var in, out int + for _, p := range s.peers { + if p.Inbound { + in++ + } else { + out++ } - // TODO: subnet-based limits - if in >= s.config.MaxInboundPeers { - return errors.New("too many inbound peers") + } + // TODO: subnet-based limits + if inbound && in >= s.config.MaxInboundPeers { + return errors.New("too many inbound peers") + } else if !inbound && out >= s.config.MaxOutboundPeers { + return errors.New("too many outbound peers") + } + return nil +} + +func (s *Syncer) alreadyConnected(peer *gateway.Peer) bool { + s.mu.Lock() + defer s.mu.Unlock() + for _, p := range s.peers { + if p.UniqueID == peer.UniqueID { + return true } - return nil } + return false +} +func (s *Syncer) acceptLoop() error { for { conn, err := s.l.Accept() if err != nil { @@ -567,10 +586,12 @@ func (s *Syncer) acceptLoop() error { } go func() { defer conn.Close() - if err := allowConnect(conn.RemoteAddr().String()); err != nil { + if err := s.allowConnect(conn.RemoteAddr().String(), true); err != nil { s.log.Printf("rejected inbound connection from %v: %v", conn.RemoteAddr(), err) } else if p, err := gateway.Accept(conn, s.header); err != nil { s.log.Printf("failed to accept inbound connection from %v: %v", conn.RemoteAddr(), err) + } else if s.alreadyConnected(p) { + s.log.Printf("rejected inbound connection from %v: already connected", conn.RemoteAddr()) } else { s.runPeer(p) } @@ -635,6 +656,11 @@ func (s *Syncer) peerLoop(closeChan <-chan struct{}) error { return false } } + closing := func() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.l == nil + } for fst := true; fst || sleep(); fst = false { if numOutbound() >= s.config.MaxOutboundPeers { continue @@ -645,7 +671,7 @@ func (s *Syncer) peerLoop(closeChan <-chan struct{}) error { continue } for _, p := range candidates { - if numOutbound() >= s.config.MaxOutboundPeers { + if numOutbound() >= s.config.MaxOutboundPeers || closing() { break } if _, err := s.Connect(p); err == nil { @@ -736,7 +762,7 @@ func (s *Syncer) syncLoop(closeChan <-chan struct{}) error { if err != nil { s.log.Printf("syncing with %v failed after %v blocks: %v", p, totalBlocks, err) } else if newTip := s.cm.Tip(); newTip != oldTip { - s.log.Printf("finished syncing %v blocks with %v, tip now %v", p, totalBlocks, newTip) + s.log.Printf("finished syncing %v blocks with %v, tip now %v", totalBlocks, p, newTip) } else { s.log.Printf("finished syncing with %v, tip unchanged", p) } @@ -761,6 +787,7 @@ func (s *Syncer) Run() error { close(closeChan) s.l.Close() s.mu.Lock() + s.l = nil for addr, p := range s.peers { p.Close() delete(s.peers, addr) @@ -776,35 +803,39 @@ func (s *Syncer) Run() error { // Connect forms an outbound connection to a peer. func (s *Syncer) Connect(addr string) (*gateway.Peer, error) { - allowConnect := func(peer string) error { - s.mu.Lock() - defer s.mu.Unlock() - if s.pm.Banned(peer) { - return errors.New("banned") - } - var out int - for _, p := range s.peers { - if !p.Inbound { - out++ - } - } - // TODO: subnet-based limits - if out >= s.config.MaxOutboundPeers { - return errors.New("too many outbound peers") - } - return nil - } - - if err := allowConnect(addr); err != nil { + if err := s.allowConnect(addr, false); err != nil { return nil, err } - conn, err := net.DialTimeout("tcp", addr, s.config.ConnectTimeout) + ctx, cancel := context.WithTimeout(context.Background(), s.config.ConnectTimeout) + defer cancel() + // slightly gross polling hack so that we shutdown quickly + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(100 * time.Millisecond): + s.mu.Lock() + if s.l == nil { + cancel() + } + s.mu.Unlock() + } + } + }() + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", addr) if err != nil { return nil, err } + conn.SetDeadline(time.Now().Add(s.config.ConnectTimeout)) + defer conn.SetDeadline(time.Time{}) p, err := gateway.Dial(conn, s.header) if err != nil { + conn.Close() return nil, err + } else if s.alreadyConnected(p) { + conn.Close() + return nil, errors.New("already connected") } go s.runPeer(p) From f84500894d4fbb21843cb53d8055290752569eb5 Mon Sep 17 00:00:00 2001 From: lukechampine Date: Sat, 21 Oct 2023 16:19:49 -0400 Subject: [PATCH 3/3] all: Update for core changes, add UPnP --- api/api_test.go | 16 ++++----- cmd/walletd/main.go | 8 +++-- cmd/walletd/node.go | 70 +++++++++++++++++++++++++++++------- go.mod | 3 +- go.sum | 6 ++-- internal/walletutil/store.go | 40 +++++++++++---------- 6 files changed, 97 insertions(+), 46 deletions(-) diff --git a/api/api_test.go b/api/api_test.go index 546ac03..335807a 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -57,11 +57,11 @@ func TestWallet(t *testing.T) { } // create wallets - dbstore, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) if err != nil { t.Fatal(err) } - cm := chain.NewManager(dbstore, tip.State) + cm := chain.NewManager(dbstore, tipState) wm := walletutil.NewEphemeralWalletManager(cm) sav := wallet.NewSeedAddressVault(wallet.NewSeed(), 0, 20) c, shutdown := runServer(cm, nil, wm) @@ -179,11 +179,11 @@ func TestV2(t *testing.T) { secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) // create wallets - dbstore, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + dbstore, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) if err != nil { t.Fatal(err) } - cm := chain.NewManager(dbstore, tip.State) + cm := chain.NewManager(dbstore, tipState) wm := walletutil.NewEphemeralWalletManager(cm) c, shutdown := runServer(cm, nil, wm) defer shutdown() @@ -383,11 +383,11 @@ func TestP2P(t *testing.T) { secondaryAddress := types.StandardUnlockHash(secondaryPrivateKey.PublicKey()) // create wallets - dbstore1, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + dbstore1, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) if err != nil { t.Fatal(err) } - cm1 := chain.NewManager(dbstore1, tip.State) + cm1 := chain.NewManager(dbstore1, tipState) wm1 := walletutil.NewEphemeralWalletManager(cm1) l1, err := net.Listen("tcp", ":0") if err != nil { @@ -413,11 +413,11 @@ func TestP2P(t *testing.T) { t.Fatal(err) } - dbstore2, tip, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) + dbstore2, tipState, err := chain.NewDBStore(chain.NewMemDB(), n, genesisBlock) if err != nil { t.Fatal(err) } - cm2 := chain.NewManager(dbstore2, tip.State) + cm2 := chain.NewManager(dbstore2, tipState) wm2 := walletutil.NewEphemeralWalletManager(cm2) l2, err := net.Listen("tcp", ":0") if err != nil { diff --git a/cmd/walletd/main.go b/cmd/walletd/main.go index c8e7b85..da11cc1 100644 --- a/cmd/walletd/main.go +++ b/cmd/walletd/main.go @@ -45,7 +45,7 @@ func check(context string, err error) { func getAPIPassword() string { apiPassword := os.Getenv("WALLETD_API_PASSWORD") if apiPassword != "" { - fmt.Println("Using WALLETD_API_PASSWORD environment variable.") + fmt.Println("env: Using WALLETD_API_PASSWORD environment variable") } else { fmt.Print("Enter API password: ") pw, err := term.ReadPassword(int(os.Stdin.Fd())) @@ -61,9 +61,11 @@ func getAPIPassword() string { func main() { log.SetFlags(0) - gatewayAddr := flag.String("addr", "localhost:9981", "p2p address to listen on") + gatewayAddr := flag.String("addr", ":9981", "p2p address to listen on") apiAddr := flag.String("http", "localhost:9980", "address to serve API on") dir := flag.String("dir", ".", "directory to store node state in") + network := flag.String("network", "mainnet", "network to connect to") + upnp := flag.Bool("upnp", true, "attempt to forward ports and discover IP with UPnP") flag.Parse() log.Println("walletd v0.1.0") @@ -79,7 +81,7 @@ func main() { log.Fatal(err) } - n, err := newNode(*gatewayAddr, *dir, true) + n, err := newNode(*gatewayAddr, *dir, *network, *upnp) if err != nil { log.Fatal(err) } diff --git a/cmd/walletd/node.go b/cmd/walletd/node.go index d7cdb2e..5cbe0ce 100644 --- a/cmd/walletd/node.go +++ b/cmd/walletd/node.go @@ -1,16 +1,23 @@ package main import ( + "context" + "errors" "log" "net" "path/filepath" + "strconv" + "time" bolt "go.etcd.io/bbolt" "go.sia.tech/core/chain" + "go.sia.tech/core/consensus" "go.sia.tech/core/gateway" + "go.sia.tech/core/types" "go.sia.tech/walletd/internal/syncerutil" "go.sia.tech/walletd/internal/walletutil" "go.sia.tech/walletd/syncer" + "lukechampine.com/upnp" ) var mainnetBootstrap = []string{ @@ -108,6 +115,11 @@ func (db *boltDB) Cancel() { db.tx = nil } +func (db *boltDB) Close() error { + db.Flush() + return db.db.Close() +} + type node struct { cm *chain.Manager s *syncer.Syncer @@ -116,40 +128,72 @@ type node struct { Start func() (stop func()) } -func newNode(addr, dir string, zen bool) (*node, error) { +func newNode(addr, dir string, chainNetwork string, useUPNP bool) (*node, error) { + var network *consensus.Network + var genesisBlock types.Block + var bootstrapPeers []string + switch chainNetwork { + case "mainnet": + network, genesisBlock = chain.Mainnet() + bootstrapPeers = mainnetBootstrap + case "zen": + network, genesisBlock = chain.TestnetZen() + bootstrapPeers = zenBootstrap + default: + return nil, errors.New("invalid network: must be one of 'mainnet' or 'zen'") + } + bdb, err := bolt.Open(filepath.Join(dir, "consensus.db"), 0600, nil) if err != nil { log.Fatal(err) } - network, genesisBlock := chain.Mainnet() - if zen { - network, genesisBlock = chain.TestnetZen() - } - dbstore, tip, err := chain.NewDBStore(&boltDB{db: bdb}, network, genesisBlock) + db := &boltDB{db: bdb} + dbstore, tipState, err := chain.NewDBStore(db, network, genesisBlock) if err != nil { return nil, err } - cm := chain.NewManager(dbstore, tip.State) + cm := chain.NewManager(dbstore, tipState) l, err := net.Listen("tcp", addr) if err != nil { return nil, err } + syncerAddr := l.Addr().String() + if useUPNP { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if d, err := upnp.Discover(ctx); err != nil { + log.Println("WARN: couldn't discover UPnP device:", err) + } else { + _, portStr, _ := net.SplitHostPort(addr) + port, _ := strconv.Atoi(portStr) + if !d.IsForwarded(uint16(port), "TCP") { + if err := d.Forward(uint16(port), "TCP", "walletd"); err != nil { + log.Println("WARN: couldn't forward port:", err) + } else { + log.Println("p2p: Forwarded port", port) + } + } + if ip, err := d.ExternalIP(); err != nil { + log.Println("WARN: couldn't determine external IP:", err) + } else { + log.Println("p2p: External IP is", ip) + syncerAddr = net.JoinHostPort(ip, portStr) + } + } + } + ps, err := syncerutil.NewJSONPeerStore(filepath.Join(dir, "peers.json")) if err != nil { log.Fatal(err) } - bootstrapPeers := mainnetBootstrap - if zen { - bootstrapPeers = zenBootstrap - } for _, peer := range bootstrapPeers { ps.AddPeer(peer) } header := gateway.Header{ GenesisID: genesisBlock.ID(), UniqueID: gateway.GenerateUniqueID(), - NetAddress: l.Addr().String(), + NetAddress: syncerAddr, } s := syncer.New(l, cm, ps, header, syncer.WithLogger(log.Default())) @@ -171,7 +215,7 @@ func newNode(addr, dir string, zen bool) (*node, error) { return func() { l.Close() <-ch - bdb.Close() + db.Close() } }, }, nil diff --git a/go.mod b/go.mod index 211885d..b843405 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,12 @@ go 1.18 require ( go.etcd.io/bbolt v1.3.7 - go.sia.tech/core v0.1.12-0.20230915021325-3ca4ff703dc6 + go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 go.sia.tech/jape v0.9.0 go.sia.tech/web/walletd v0.9.0 golang.org/x/term v0.6.0 lukechampine.com/frand v1.4.2 + lukechampine.com/upnp v0.2.0 ) require ( diff --git a/go.sum b/go.sum index 19b938b..8c02554 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= -go.sia.tech/core v0.1.12-0.20230915021325-3ca4ff703dc6 h1:WYQDfzDWBcbr9t+nnatOShOUB4aD2XSu/UGz28XdCwU= -go.sia.tech/core v0.1.12-0.20230915021325-3ca4ff703dc6/go.mod h1:D17UWSn99SEfQnEaR9G9n6Kz9+BwqMoUgZ6Cl424LsQ= +go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0 h1:2nKOKa99g9h9m3hL5UortAbmnwuwXhDcTHIhzmqBae8= +go.sia.tech/core v0.1.12-0.20231021194448-f1e65eb9f0d0/go.mod h1:3EoY+rR78w1/uGoXXVqcYdwSjSJKuEMI5bL7WROA27Q= go.sia.tech/jape v0.9.0 h1:kWgMFqALYhLMJYOwWBgJda5ko/fi4iZzRxHRP7pp8NY= go.sia.tech/jape v0.9.0/go.mod h1:4QqmBB+t3W7cNplXPj++ZqpoUb2PeiS66RLpXmEGap4= go.sia.tech/mux v1.2.0 h1:ofa1Us9mdymBbGMY2XH/lSpY8itFsKIo/Aq8zwe+GHU= @@ -30,3 +30,5 @@ golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= lukechampine.com/frand v1.4.2 h1:RzFIpOvkMXuPMBb9maa4ND4wjBn71E1Jpf8BzJHMaVw= lukechampine.com/frand v1.4.2/go.mod h1:4S/TM2ZgrKejMcKMbeLjISpJMO+/eZ1zu3vYX9dtj3s= +lukechampine.com/upnp v0.2.0 h1:FWgYN50s8cTc5BQixLHy9uNgryw+qf3dd+oPS0I+vPQ= +lukechampine.com/upnp v0.2.0/go.mod h1:sOuF+fGSDKjpUm6QI0mfb82ScRrhj8bsqsD78O5nK1k= diff --git a/internal/walletutil/store.go b/internal/walletutil/store.go index 7e68293..0340464 100644 --- a/internal/walletutil/store.go +++ b/internal/walletutil/store.go @@ -209,16 +209,6 @@ func (s *EphemeralStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, _ bool) events := wallet.AppliedEvents(cau.State, cau.Block, cau, s.ownsAddress) s.events = append(s.events, events...) - // update proofs - for id, sce := range s.sces { - cau.UpdateElementProof(&sce.StateElement) - s.sces[id] = sce - } - for id, sfe := range s.sfes { - cau.UpdateElementProof(&sfe.StateElement) - s.sfes[id] = sfe - } - // add/remove outputs cau.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { if s.ownsAddress(sce.SiacoinOutput.Address) { @@ -241,6 +231,16 @@ func (s *EphemeralStore) ProcessChainApplyUpdate(cau *chain.ApplyUpdate, _ bool) } }) + // update proofs + for id, sce := range s.sces { + cau.UpdateElementProof(&sce.StateElement) + s.sces[id] = sce + } + for id, sfe := range s.sfes { + cau.UpdateElementProof(&sfe.StateElement) + s.sfes[id] = sfe + } + s.tip = cau.State.Index return nil } @@ -254,15 +254,6 @@ func (s *EphemeralStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error numEvents := len(wallet.AppliedEvents(cru.State, cru.Block, cru, s.ownsAddress)) s.events = s.events[:len(s.events)-numEvents] - for id, sce := range s.sces { - cru.UpdateElementProof(&sce.StateElement) - s.sces[id] = sce - } - for id, sfe := range s.sfes { - cru.UpdateElementProof(&sfe.StateElement) - s.sfes[id] = sfe - } - cru.ForEachSiacoinElement(func(sce types.SiacoinElement, spent bool) { if s.ownsAddress(sce.SiacoinOutput.Address) { if !spent { @@ -283,6 +274,17 @@ func (s *EphemeralStore) ProcessChainRevertUpdate(cru *chain.RevertUpdate) error } } }) + + // update proofs + for id, sce := range s.sces { + cru.UpdateElementProof(&sce.StateElement) + s.sces[id] = sce + } + for id, sfe := range s.sfes { + cru.UpdateElementProof(&sfe.StateElement) + s.sfes[id] = sfe + } + s.tip = cru.State.Index return nil }