From d50e7df4c1c5cf18b0290e2fb2d96c71b2231c0b Mon Sep 17 00:00:00 2001 From: maria jose Date: Fri, 14 Apr 2023 14:20:52 -0400 Subject: [PATCH] e2etest: more refactoring * add helper methods * helpers remove unnecessary code * main, add election base struct and remove code that already exists in helpers * update getFaucetPackage calls * use helper methods in plaintext test * remove unnecessary code * update generateProofs * update plaintextelection * update generateProofs for correctly support anonymous voting proofs * port anonelection to use helpers * port encryptedelection to use helpers * cleaning * main sort imports * wait longer in some places * improve approach in Setup with a flexible setupElection --- cmd/end2endtest/account.go | 2 +- cmd/end2endtest/encrypted.go | 230 ++--------------------- cmd/end2endtest/helpers.go | 326 +++++++++++++++++++++++++++++++++ cmd/end2endtest/main.go | 26 ++- cmd/end2endtest/plaintext.go | 265 ++++----------------------- cmd/end2endtest/zkweighted.go | 336 ++++++++-------------------------- 6 files changed, 462 insertions(+), 723 deletions(-) create mode 100644 cmd/end2endtest/helpers.go diff --git a/cmd/end2endtest/account.go b/cmd/end2endtest/account.go index f4a748688..63503220c 100644 --- a/cmd/end2endtest/account.go +++ b/cmd/end2endtest/account.go @@ -57,7 +57,7 @@ func (t *E2ETokenTxs) Setup(api *apiclient.HTTPclient, config *config) error { } // get faucet package for alice - t.aliceFP, err = getFaucetPackage(t.config, t.alice.Address().Hex()) + t.aliceFP, err = getFaucetPackage(t.config.faucet, t.config.faucetAuthToken, t.alice.Address().Hex()) if err != nil { return err } diff --git a/cmd/end2endtest/encrypted.go b/cmd/end2endtest/encrypted.go index 7c2b32a5e..33d0a8ff3 100644 --- a/cmd/end2endtest/encrypted.go +++ b/cmd/end2endtest/encrypted.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "fmt" - "math/big" "os" "strings" "sync" @@ -15,9 +14,6 @@ import ( "go.vocdoni.io/dvote/apiclient" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" - "go.vocdoni.io/dvote/types" - "go.vocdoni.io/dvote/util" - "go.vocdoni.io/proto/build/go/models" ) func init() { @@ -31,234 +27,30 @@ func init() { var _ VochainTest = (*E2EEncryptedElection)(nil) type E2EEncryptedElection struct { - api *apiclient.HTTPclient - config *config - - election *vapi.Election - voterAccounts []*ethereum.SignKeys - proofs map[string]*apiclient.CensusProof + e2eElection } func (t *E2EEncryptedElection) Setup(api *apiclient.HTTPclient, c *config) error { t.api = api t.config = c - // Set the account in the API client, so we can sign transactions - if err := api.SetAccount(hex.EncodeToString(c.accountKeys[0].PrivateKey())); err != nil { - log.Fatal(err) - } - - // If the account does not exist, create a new one - // TODO: check if the account balance is low and use the faucet - acc, err := api.Account("") - if err != nil { - log.Infof("getting faucet package") - faucetPkg, err := getFaucetPackage(c, api.MyAddress().Hex()) - if err != nil { - log.Fatal(err) - } - - // Create the organization account and bootstraping with the faucet package - log.Infof("creating Vocdoni account %s", api.MyAddress().Hex()) - log.Debugf("faucetPackage is %x", faucetPkg) - hash, err := api.AccountBootstrap(faucetPkg, &vapi.AccountMetadata{ - Name: map[string]string{"default": "test account " + api.MyAddress().Hex()}, - Description: map[string]string{"default": "test description"}, - Version: "1.0", - }) - if err != nil { - log.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - if _, err := api.WaitUntilTxIsMined(ctx, hash); err != nil { - log.Fatalf("gave up waiting for tx %x to be mined: %s", hash, err) - } - - acc, err = api.Account("") - if err != nil { - log.Fatal(err) - } - if c.faucet != "" && acc.Balance == 0 { - log.Fatal("account balance is 0") - } - } - - log.Infof("account %s balance is %d", api.MyAddress().Hex(), acc.Balance) - - // Create a new census - censusID, err := api.NewCensus(vapi.CensusTypeWeighted) - if err != nil { - log.Fatal(err) - } - log.Infof("new census created with id %s", censusID.String()) - - // Generate 10 participant accounts - t.voterAccounts = ethereum.NewSignKeysBatch(c.nvotes) - - // Add the accounts to the census by batches - participants := &vapi.CensusParticipants{} - for i, voterAccount := range t.voterAccounts { - participants.Participants = append(participants.Participants, - vapi.CensusParticipant{ - Key: voterAccount.Address().Bytes(), - Weight: (*types.BigInt)(new(big.Int).SetUint64(10)), - }) - if i == len(t.voterAccounts)-1 || ((i+1)%vapi.MaxCensusAddBatchSize == 0) { - if err := api.CensusAddParticipants(censusID, participants); err != nil { - log.Fatal(err) - } - log.Infof("added %d participants to census %s", - len(participants.Participants), censusID.String()) - participants = &vapi.CensusParticipants{} - } - } - - // Check census size - size, err := api.CensusSize(censusID) - if err != nil { - log.Fatal(err) - } - if size != uint64(c.nvotes) { - log.Fatalf("census size is %d, expected %d", size, c.nvotes) - } - log.Infof("census %s size is %d", censusID.String(), size) - - // Publish the census - root, censusURI, err := api.CensusPublish(censusID) - if err != nil { - log.Fatal(err) - } - log.Infof("census published with root %s", root.String()) - - // Check census size (of the published census) - size, err = api.CensusSize(root) - if err != nil { - log.Fatal(err) - } - if size != uint64(c.nvotes) { - log.Fatalf("published census size is %d, expected %d", size, c.nvotes) - } - - // Create a new Election - electionID, err := api.NewElection(&vapi.ElectionDescription{ - Title: map[string]string{"default": fmt.Sprintf("Test election %s", util.RandomHex(8))}, - Description: map[string]string{"default": "Test election description"}, - EndDate: time.Now().Add(time.Minute * 20), - - VoteType: vapi.VoteType{ - UniqueChoices: false, - MaxVoteOverwrites: 1, - }, - - ElectionType: vapi.ElectionType{ - Autostart: true, - Interruptible: true, - Anonymous: false, - SecretUntilTheEnd: true, - DynamicCensus: false, - }, - - Census: vapi.CensusTypeDescription{ - RootHash: root, - URL: censusURI, - Type: "weighted", - Size: uint64(len(t.voterAccounts)), - }, - - Questions: []vapi.Question{ - { - Title: map[string]string{"default": "Test question 1"}, - Description: map[string]string{"default": "Test question 1 description"}, - Choices: []vapi.ChoiceMetadata{ - { - Title: map[string]string{"default": "Yes"}, - Value: 0, - }, - { - Title: map[string]string{"default": "No"}, - Value: 1, - }, - }, - }, - }, - }) - if err != nil { - log.Fatal(err) + ed := newTestElectionDescription() + ed.ElectionType = vapi.ElectionType{ + Autostart: true, + Interruptible: true, + SecretUntilTheEnd: true, } - log.Infof("created new election with id %s", electionID.String()) + ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} + ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} - t.proofs = t.votingProofs(root, t.voterAccounts) - - // Wait for the election to start - ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - t.election, err = api.WaitUntilElectionStarts(ctx, electionID) - if err != nil { - log.Fatal(err) + if err := t.setupElection(ed); err != nil { + return err } log.Debugf("election details: %+v", *t.election) - return nil } -// votingProofs generates the voting proofs (parallelized) -func (t *E2EEncryptedElection) votingProofs(root types.HexBytes, - voterAccounts []*ethereum.SignKeys) map[string]*apiclient.CensusProof { - type voterProof struct { - proof *apiclient.CensusProof - address string - } - proofs := make(map[string]*apiclient.CensusProof, t.config.nvotes) - proofCh := make(chan *voterProof) - stopProofs := make(chan bool) - go func() { - for { - select { - case p := <-proofCh: - proofs[p.address] = p.proof - case <-stopProofs: - return - } - } - }() - - addNaccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { - defer wg.Done() - log.Infof("generating %d voting proofs", len(accounts)) - for _, acc := range accounts { - pr, err := t.api.CensusGenProof(root, acc.Address().Bytes()) - if err != nil { - log.Fatal(err) - } - pr.KeyType = models.ProofArbo_ADDRESS - proofCh <- &voterProof{ - proof: pr, - address: acc.Address().Hex(), - } - } - } - - pcount := t.config.nvotes / t.config.parallelCount - var wg sync.WaitGroup - for i := 0; i < len(t.voterAccounts); i += pcount { - end := i + pcount - if end > len(t.voterAccounts) { - end = len(t.voterAccounts) - } - wg.Add(1) - go addNaccounts(t.voterAccounts[i:end], &wg) - } - - wg.Wait() - time.Sleep(time.Second) // wait a grace time for the last proof to be added - log.Debugf("%d/%d voting proofs generated successfully", len(proofs), len(t.voterAccounts)) - stopProofs <- true - - return proofs -} - func (t *E2EEncryptedElection) Teardown() error { // nothing to do here return nil @@ -362,7 +154,7 @@ func (t *E2EEncryptedElection) Run() error { } // Wait for the election to be in RESULTS state - ctx, cancel := context.WithTimeout(context.Background(), apiclient.WaitTimeout) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*300) defer cancel() election, err := api.WaitUntilElectionStatus(ctx, t.election.ElectionID, "RESULTS") if err != nil { diff --git a/cmd/end2endtest/helpers.go b/cmd/end2endtest/helpers.go new file mode 100644 index 000000000..6696972a5 --- /dev/null +++ b/cmd/end2endtest/helpers.go @@ -0,0 +1,326 @@ +package main + +import ( + "context" + "encoding/hex" + "fmt" + "math/big" + "sync" + "time" + + vapi "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/apiclient" + "go.vocdoni.io/dvote/crypto/ethereum" + "go.vocdoni.io/dvote/crypto/zk" + "go.vocdoni.io/dvote/log" + "go.vocdoni.io/dvote/types" + "go.vocdoni.io/dvote/util" + "go.vocdoni.io/proto/build/go/models" +) + +func newTestElectionDescription() *vapi.ElectionDescription { + return &vapi.ElectionDescription{ + Title: map[string]string{"default": fmt.Sprintf("Test election %s", util.RandomHex(8))}, + Description: map[string]string{"default": "Test election description"}, + EndDate: time.Now().Add(time.Minute * 20), + + Questions: []vapi.Question{ + { + Title: map[string]string{"default": "Test question 1"}, + Description: map[string]string{"default": "Test question 1 description"}, + Choices: []vapi.ChoiceMetadata{ + { + Title: map[string]string{"default": "Yes"}, + Value: 0, + }, + { + Title: map[string]string{"default": "No"}, + Value: 1, + }, + }, + }, + }, + } +} + +func (t e2eElection) createAccount(address string) (*vapi.Account, error) { + faucetPkg, err := getFaucetPackage(t.config.faucet, t.config.faucetAuthToken, address) + if err != nil { + return nil, err + } + + accountMetadata := &vapi.AccountMetadata{ + Name: map[string]string{"default": "test account " + address}, + Description: map[string]string{"default": "test description"}, + Version: "1.0", + } + + log.Infof("creating Vocdoni account %s", address) + hash, err := t.api.AccountBootstrap(faucetPkg, accountMetadata) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) + defer cancel() + + if _, err := t.api.WaitUntilTxIsMined(ctx, hash); err != nil { + log.Errorf("gave up waiting for tx %x to be mined: %s", hash, err) + return nil, err + } + + // check the account + acc, err := t.api.Account("") + if err != nil { + return nil, err + } + if t.config.faucet != "" && acc.Balance == 0 { + log.Error("account balance is 0") + return nil, err + } + return acc, nil + +} + +func (t e2eElection) addParticipantsCensus(censusType string, censusID types.HexBytes) error { + participants := &vapi.CensusParticipants{} + + for i, voterAccount := range t.voterAccounts { + keyAddr, err := getCensusParticipantKey(voterAccount, censusType) + if err != nil { + return err + } + participants.Participants = append(participants.Participants, + vapi.CensusParticipant{ + Key: keyAddr, + Weight: (*types.BigInt)(new(big.Int).SetUint64(10)), + }) + + if i == len(t.voterAccounts)-1 || ((i+1)%vapi.MaxCensusAddBatchSize == 0) { + if err := t.api.CensusAddParticipants(censusID, participants); err != nil { + return err + } + + log.Infof("added %d participants to census %s", + len(participants.Participants), censusID.String()) + + participants = &vapi.CensusParticipants{} + } + } + return nil +} + +func (t e2eElection) isCensusSizeValid(censusID types.HexBytes) bool { + size, err := t.api.CensusSize(censusID) + if err != nil { + log.Errorf("unable to get census size from api") + return false + } + if size != uint64(t.config.nvotes) { + log.Errorf("census size is %d, expected %d", size, t.config.nvotes) + return false + } + log.Infof("census %s size is %d", censusID.String(), size) + return true +} + +func (t e2eElection) createElection(electionDescrip *vapi.ElectionDescription) (*vapi.Election, error) { + electionID, err := t.api.NewElection(electionDescrip) + if err != nil { + return nil, err + } + + log.Infof("created new election with id %s - now wait until it starts", electionID.String()) + + // Wait for the election to start + ctx, cancel := context.WithTimeout(context.Background(), time.Second*80) + defer cancel() + election, err := t.api.WaitUntilElectionStarts(ctx, electionID) + if err != nil { + return nil, err + } + election.ElectionID = electionID + + return election, nil +} + +func (t e2eElection) generateProofs(root types.HexBytes, isAnonymousVoting bool) map[string]*apiclient.CensusProof { + type voterProof struct { + proof *apiclient.CensusProof + address string + } + proofs := make(map[string]*apiclient.CensusProof, t.config.nvotes) + proofCh := make(chan *voterProof) + stopProofs := make(chan bool) + go func() { + for { + select { + case p := <-proofCh: + proofs[p.address] = p.proof + case <-stopProofs: + return + } + } + }() + + apiClientMtx := &sync.Mutex{} + addNaccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { + defer wg.Done() + log.Infof("generating %d voting proofs", len(accounts)) + for _, acc := range accounts { + voterKey := acc.Address().Bytes() + + if isAnonymousVoting { + apiClientMtx.Lock() + privKey := acc.PrivateKey() + if err := t.api.SetAccount(privKey.String()); err != nil { + apiClientMtx.Unlock() + log.Fatal(err) + return + } + voterKey = t.api.MyZkAddress().Bytes() + } + + pr, err := t.api.CensusGenProof(root, voterKey) + if isAnonymousVoting { + apiClientMtx.Unlock() + } + if err != nil { + log.Fatal(err) + } + + if !isAnonymousVoting { + pr.KeyType = models.ProofArbo_ADDRESS + } + proofCh <- &voterProof{ + proof: pr, + address: acc.Address().Hex(), + } + } + } + + pcount := t.config.nvotes / t.config.parallelCount + var wg sync.WaitGroup + for i := 0; i < len(t.voterAccounts); i += pcount { + end := i + pcount + if end > len(t.voterAccounts) { + end = len(t.voterAccounts) + } + wg.Add(1) + go addNaccounts(t.voterAccounts[i:end], &wg) + } + + wg.Wait() + time.Sleep(time.Second) // wait a grace time for the last proof to be added + log.Debugf("%d/%d voting proofs generated successfully", len(proofs), len(t.voterAccounts)) + stopProofs <- true + + return proofs +} + +func (t *e2eElection) setupElection(ed *vapi.ElectionDescription) error { + // Set the account in the API client, so we can sign transactions + if err := t.api.SetAccount(hex.EncodeToString(t.config.accountKeys[0].PrivateKey())); err != nil { + return err + } + + // If the account does not exist, create a new one + // TODO: check if the account balance is low and use the faucet + acc, err := t.api.Account("") + if err != nil { + acc, err = t.createAccount(t.api.MyAddress().Hex()) + if err != nil { + fmt.Println("error account1") + return err + } + } + log.Infof("account %s balance is %d", t.api.MyAddress().Hex(), acc.Balance) + + // Create a new census + censusID, err := t.api.NewCensus(ed.Census.Type) + if err != nil { + fmt.Println("error new census") + return err + } + log.Infof("new census created with id %s", censusID.String()) + + // Generate 10 participant accounts + t.voterAccounts = ethereum.NewSignKeysBatch(t.config.nvotes) + + // Add the accounts to the census by batches + if err := t.addParticipantsCensus(ed.Census.Type, censusID); err != nil { + fmt.Println("error addding participants") + return err + } + + // Check census size + if !t.isCensusSizeValid(censusID) { + fmt.Println("error is size valid") + return err + } + + // Publish the census + root, censusURI, err := t.api.CensusPublish(censusID) + if err != nil { + fmt.Println("error census publish") + return err + } + log.Infof("census published with root %s", root.String()) + ed.Census.RootHash = root + ed.Census.URL = censusURI + ed.Census.Size = uint64(t.config.nvotes) + + // Check census size (of the published census) + if !t.isCensusSizeValid(root) { + fmt.Println("error is size valid 2") + return err + } + + t.election, err = t.createElection(ed) + if err != nil { + fmt.Println("error create election") + return err + } + + log.Infof("created new election with id %s", t.election.ElectionID.String()) + + // Wait for the election to start + ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) + defer cancel() + t.election, err = t.api.WaitUntilElectionStarts(ctx, t.election.ElectionID) + if err != nil { + fmt.Println("error wait until") + return err + } + + t.proofs = t.generateProofs(root, ed.ElectionType.Anonymous) + + return nil +} + +func getFaucetPackage(faucet, faucetAuthToken, myAddress string) (*models.FaucetPackage, error) { + switch faucet { + case "": + return nil, fmt.Errorf("need to pass a valid --faucet") + case "dev": + return apiclient.GetFaucetPackageFromDevService(myAddress) + default: + return apiclient.GetFaucetPackageFromRemoteService(faucet+myAddress, faucetAuthToken) + } +} + +func getCensusParticipantKey(voterAccount *ethereum.SignKeys, censusType string) ([]byte, error) { + var key []byte + + switch censusType { + case vapi.CensusTypeWeighted: + key = voterAccount.Address().Bytes() + case vapi.CensusTypeZKWeighted: + zkAddr, err := zk.AddressFromSignKeys(voterAccount) + if err != nil { + return nil, err + } + key = zkAddr.Bytes() + } + return key, nil +} diff --git a/cmd/end2endtest/main.go b/cmd/end2endtest/main.go index b5cea63f6..b18a51035 100644 --- a/cmd/end2endtest/main.go +++ b/cmd/end2endtest/main.go @@ -9,11 +9,10 @@ import ( "github.com/google/uuid" flag "github.com/spf13/pflag" + vapi "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/apiclient" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/internal" - "go.vocdoni.io/proto/build/go/models" - - "go.vocdoni.io/dvote/apiclient" "go.vocdoni.io/dvote/log" "go.vocdoni.io/dvote/util" ) @@ -35,6 +34,15 @@ type VochainTest interface { Teardown() error } +type e2eElection struct { + api *apiclient.HTTPclient + config *config + + election *vapi.Election + voterAccounts []*ethereum.SignKeys + proofs map[string]*apiclient.CensusProof +} + type operation struct { test VochainTest @@ -176,15 +184,3 @@ func privKeyToSigner(key string) (*ethereum.SignKeys, error) { } return skey, nil } - -func getFaucetPackage(c *config, account string) (*models.FaucetPackage, error) { - if c.faucet == "" { - return nil, fmt.Errorf("need to pass a valid --faucet") - } - if c.faucet == "dev" { - return apiclient.GetFaucetPackageFromDevService(account) - } else { - return apiclient.GetFaucetPackageFromRemoteService(c.faucet+account, - c.faucetAuthToken) - } -} diff --git a/cmd/end2endtest/plaintext.go b/cmd/end2endtest/plaintext.go index 75b37348d..f60372647 100644 --- a/cmd/end2endtest/plaintext.go +++ b/cmd/end2endtest/plaintext.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "errors" "fmt" - "math/big" "os" "strings" "sync" @@ -15,9 +14,6 @@ import ( "go.vocdoni.io/dvote/apiclient" "go.vocdoni.io/dvote/crypto/ethereum" "go.vocdoni.io/dvote/log" - "go.vocdoni.io/dvote/types" - "go.vocdoni.io/dvote/util" - "go.vocdoni.io/proto/build/go/models" ) func init() { @@ -31,13 +27,26 @@ func init() { var _ VochainTest = (*E2EPlaintextElection)(nil) type E2EPlaintextElection struct { - api *apiclient.HTTPclient - config *config + e2eElection } -func (t *E2EPlaintextElection) Setup(api *apiclient.HTTPclient, config *config) error { +func (t *E2EPlaintextElection) Setup(api *apiclient.HTTPclient, c *config) error { t.api = api - t.config = config + t.config = c + + ed := newTestElectionDescription() + ed.ElectionType = vapi.ElectionType{ + Autostart: true, + Interruptible: true, + } + ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} + ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeWeighted} + + if err := t.setupElection(ed); err != nil { + return err + } + + log.Debugf("election details: %+v", *t.election) return nil } @@ -50,215 +59,9 @@ func (t *E2EPlaintextElection) Run() error { c := t.config api := t.api - // Set the account in the API client, so we can sign transactions - if err := api.SetAccount(hex.EncodeToString(c.accountKeys[0].PrivateKey())); err != nil { - log.Fatal(err) - } - - // If the account does not exist, create a new one - // TODO: check if the account balance is low and use the faucet - acc, err := api.Account("") - if err != nil { - log.Infof("getting faucet package") - faucetPkg, err := getFaucetPackage(c, api.MyAddress().Hex()) - if err != nil { - log.Fatal(err) - } - - // Create the organization account and bootstraping with the faucet package - log.Infof("creating Vocdoni account %s", api.MyAddress().Hex()) - log.Debugf("faucetPackage is %x", faucetPkg) - hash, err := api.AccountBootstrap(faucetPkg, &vapi.AccountMetadata{ - Name: map[string]string{"default": "test account " + api.MyAddress().Hex()}, - Description: map[string]string{"default": "test description"}, - Version: "1.0", - }) - if err != nil { - log.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - if _, err := api.WaitUntilTxIsMined(ctx, hash); err != nil { - log.Fatalf("gave up waiting for tx %x to be mined: %s", hash, err) - } - - acc, err = api.Account("") - if err != nil { - log.Fatal(err) - } - if c.faucet != "" && acc.Balance == 0 { - log.Fatal("account balance is 0") - } - } - - log.Infof("account %s balance is %d", api.MyAddress().Hex(), acc.Balance) - - // Create a new census - censusID, err := api.NewCensus("weighted") - if err != nil { - log.Fatal(err) - } - log.Infof("new census created with id %s", censusID.String()) - - // Generate 10 participant accounts - voterAccounts := ethereum.NewSignKeysBatch(c.nvotes) - - // Add the accounts to the census by batches - participants := &vapi.CensusParticipants{} - for i, voterAccount := range voterAccounts { - participants.Participants = append(participants.Participants, - vapi.CensusParticipant{ - Key: voterAccount.Address().Bytes(), - Weight: (*types.BigInt)(new(big.Int).SetUint64(10)), - }) - if i == len(voterAccounts)-1 || ((i+1)%vapi.MaxCensusAddBatchSize == 0) { - if err := api.CensusAddParticipants(censusID, participants); err != nil { - log.Fatal(err) - } - log.Infof("added %d participants to census %s", - len(participants.Participants), censusID.String()) - participants = &vapi.CensusParticipants{} - } - } - - // Check census size - size, err := api.CensusSize(censusID) - if err != nil { - log.Fatal(err) - } - if size != uint64(c.nvotes) { - log.Fatalf("census size is %d, expected %d", size, c.nvotes) - } - log.Infof("census %s size is %d", censusID.String(), size) - - // Publish the census - root, censusURI, err := api.CensusPublish(censusID) - if err != nil { - log.Fatal(err) - } - log.Infof("census published with root %s", root.String()) - - // Check census size (of the published census) - size, err = api.CensusSize(root) - if err != nil { - log.Fatal(err) - } - if size != uint64(c.nvotes) { - log.Fatalf("published census size is %d, expected %d", size, c.nvotes) - } - - // Generate the voting proofs (parallelized) - type voterProof struct { - proof *apiclient.CensusProof - address string - } - proofs := make(map[string]*apiclient.CensusProof, c.nvotes) - proofCh := make(chan *voterProof) - stopProofs := make(chan bool) - go func() { - for { - select { - case p := <-proofCh: - proofs[p.address] = p.proof - case <-stopProofs: - return - } - } - }() - - addNaccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { - defer wg.Done() - log.Infof("generating %d voting proofs", len(accounts)) - for _, acc := range accounts { - pr, err := api.CensusGenProof(root, acc.Address().Bytes()) - if err != nil { - log.Fatal(err) - } - pr.KeyType = models.ProofArbo_ADDRESS - proofCh <- &voterProof{ - proof: pr, - address: acc.Address().Hex(), - } - } - } - - pcount := c.nvotes / c.parallelCount - var wg sync.WaitGroup - for i := 0; i < len(voterAccounts); i += pcount { - end := i + pcount - if end > len(voterAccounts) { - end = len(voterAccounts) - } - wg.Add(1) - go addNaccounts(voterAccounts[i:end], &wg) - } - - wg.Wait() - time.Sleep(time.Second) // wait a grace time for the last proof to be added - log.Debugf("%d/%d voting proofs generated successfully", len(proofs), len(voterAccounts)) - stopProofs <- true - - // Create a new Election - electionID, err := api.NewElection(&vapi.ElectionDescription{ - Title: map[string]string{"default": fmt.Sprintf("Test election %s", util.RandomHex(8))}, - Description: map[string]string{"default": "Test election description"}, - EndDate: time.Now().Add(time.Minute * 20), - - VoteType: vapi.VoteType{ - UniqueChoices: false, - MaxVoteOverwrites: 1, - }, - - ElectionType: vapi.ElectionType{ - Autostart: true, - Interruptible: true, - Anonymous: false, - SecretUntilTheEnd: false, - DynamicCensus: false, - }, - - Census: vapi.CensusTypeDescription{ - RootHash: root, - URL: censusURI, - Type: "weighted", - Size: uint64(len(voterAccounts)), - }, - - Questions: []vapi.Question{ - { - Title: map[string]string{"default": "Test question 1"}, - Description: map[string]string{"default": "Test question 1 description"}, - Choices: []vapi.ChoiceMetadata{ - { - Title: map[string]string{"default": "Yes"}, - Value: 0, - }, - { - Title: map[string]string{"default": "No"}, - Value: 1, - }, - }, - }, - }, - }) - if err != nil { - log.Fatal(err) - } - log.Infof("created new election with id %s - now wait until it starts", electionID.String()) - - // Wait for the election to start - ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - election, err := api.WaitUntilElectionStarts(ctx, electionID) - if err != nil { - log.Fatal(err) - } - - log.Debugf("election details: %+v", *election) - // Send the votes (parallelized) startTime := time.Now() - wg = sync.WaitGroup{} + wg := sync.WaitGroup{} voteAccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { defer wg.Done() log.Infof("sending %d votes", len(accounts)) @@ -275,8 +78,8 @@ func (t *E2EPlaintextElection) Run() error { for i, voterAccount := range accountsMap { c := api.Clone(fmt.Sprintf("%x", voterAccount.PrivateKey())) _, err := c.Vote(&apiclient.VoteData{ - ElectionID: electionID, - ProofMkTree: proofs[voterAccount.Address().Hex()], + ElectionID: t.election.ElectionID, + ProofMkTree: t.proofs[voterAccount.Address().Hex()], Choices: []int{i % 2}, }) // if the context deadline is reached, we don't need to print it (let's jus retry) @@ -303,14 +106,14 @@ func (t *E2EPlaintextElection) Run() error { time.Sleep(time.Second * 2) } - pcount = c.nvotes / c.parallelCount - for i := 0; i < len(voterAccounts); i += pcount { + pcount := c.nvotes / c.parallelCount + for i := 0; i < len(t.voterAccounts); i += pcount { end := i + pcount - if end > len(voterAccounts) { - end = len(voterAccounts) + if end > len(t.voterAccounts) { + end = len(t.voterAccounts) } wg.Add(1) - go voteAccounts(voterAccounts[i:end], &wg) + go voteAccounts(t.voterAccounts[i:end], &wg) } wg.Wait() @@ -320,7 +123,7 @@ func (t *E2EPlaintextElection) Run() error { // Wait for all the votes to be verified log.Infof("waiting for all the votes to be registered...") for { - count, err := api.ElectionVoteCount(electionID) + count, err := api.ElectionVoteCount(t.election.ElectionID) if err != nil { log.Warn(err) } @@ -344,36 +147,36 @@ func (t *E2EPlaintextElection) Run() error { // End the election by setting the status to ENDED log.Infof("ending election...") - hash, err := api.SetElectionStatus(electionID, "ENDED") + hash, err := api.SetElectionStatus(t.election.ElectionID, "ENDED") if err != nil { log.Fatal(err) } // Check the election status is actually ENDED - ctx, cancel = context.WithTimeout(context.Background(), time.Second*40) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) defer cancel() if _, err := api.WaitUntilTxIsMined(ctx, hash); err != nil { log.Fatalf("gave up waiting for tx %s to be mined: %s", hash, err) } - election, err = api.Election(electionID) + t.election, err = api.Election(t.election.ElectionID) if err != nil { log.Fatal(err) } - if election.Status != "ENDED" { + if t.election.Status != "ENDED" { log.Fatal("election status is not ENDED") } - log.Infof("election %s status is ENDED", electionID.String()) + log.Infof("election %s status is ENDED", t.election.ElectionID.String()) // Wait for the election to be in RESULTS state ctx, cancel = context.WithTimeout(context.Background(), time.Second*300) defer cancel() - election, err = api.WaitUntilElectionStatus(ctx, electionID, "RESULTS") + t.election, err = api.WaitUntilElectionStatus(ctx, t.election.ElectionID, "RESULTS") if err != nil { log.Fatal(err) } - log.Infof("election %s status is RESULTS", electionID.String()) - log.Infof("election results: %v", election.Results) + log.Infof("election %s status is RESULTS", t.election.ElectionID.String()) + log.Infof("election results: %v", t.election.Results) return nil } diff --git a/cmd/end2endtest/zkweighted.go b/cmd/end2endtest/zkweighted.go index 6beafe0b8..6c882a859 100644 --- a/cmd/end2endtest/zkweighted.go +++ b/cmd/end2endtest/zkweighted.go @@ -2,7 +2,7 @@ package main import ( "context" - "fmt" + "errors" "math/big" "os" "strings" @@ -12,11 +12,7 @@ import ( vapi "go.vocdoni.io/dvote/api" "go.vocdoni.io/dvote/apiclient" "go.vocdoni.io/dvote/crypto/ethereum" - "go.vocdoni.io/dvote/crypto/zk" "go.vocdoni.io/dvote/log" - "go.vocdoni.io/dvote/types" - "go.vocdoni.io/dvote/util" - "go.vocdoni.io/proto/build/go/models" ) func init() { @@ -30,13 +26,26 @@ func init() { var _ VochainTest = (*E2EAnonElection)(nil) type E2EAnonElection struct { - api *apiclient.HTTPclient - config *config + e2eElection } -func (t *E2EAnonElection) Setup(api *apiclient.HTTPclient, config *config) error { +func (t *E2EAnonElection) Setup(api *apiclient.HTTPclient, c *config) error { t.api = api - t.config = config + t.config = c + + ed := newTestElectionDescription() + ed.ElectionType = vapi.ElectionType{ + Autostart: true, + Interruptible: true, + Anonymous: true, + } + ed.VoteType = vapi.VoteType{MaxVoteOverwrites: 1} + ed.Census = vapi.CensusTypeDescription{Type: vapi.CensusTypeZKWeighted} + + if err := t.setupElection(ed); err != nil { + return err + } + log.Debugf("election details: %+v", *t.election) return nil } @@ -49,269 +58,82 @@ func (t *E2EAnonElection) Run() error { c := t.config api := t.api - // Set the account in the API client, so we can sign transactions - if err := api.SetAccount(c.accountPrivKeys[0]); err != nil { - log.Fatal(err) - } - - // If the account does not exist, create a new one - // TODO: check if the account balance is low and use the faucet - acc, err := api.Account("") - if err != nil { - var faucetPkg *models.FaucetPackage - if c.faucet != "" { - // Get the faucet package of bootstrap tokens - log.Infof("getting faucet package") - if c.faucet == "dev" { - faucetPkg, err = apiclient.GetFaucetPackageFromDevService(api.MyAddress().Hex()) - } else { - faucetPkg, err = apiclient.GetFaucetPackageFromRemoteService(c.faucet+api.MyAddress().Hex(), c.faucetAuthToken) - } - - if err != nil { - log.Fatal(err) - } - } - // Create the organization account and bootstraping with the faucet package - log.Infof("creating Vocdoni account %s", api.MyAddress().Hex()) - log.Debugf("faucetPackage is %x", faucetPkg) - hash, err := api.AccountBootstrap(faucetPkg, &vapi.AccountMetadata{ - Name: map[string]string{"default": "test account " + api.MyAddress().Hex()}, - Description: map[string]string{"default": "test description"}, - Version: "1.0", - }) - if err != nil { - log.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - if _, err := api.WaitUntilTxIsMined(ctx, hash); err != nil { - log.Fatalf("gave up waiting for tx %x to be mined: %s", hash, err) - } - - acc, err = api.Account("") - if err != nil { - log.Fatal(err) - } - if c.faucet != "" && acc.Balance == 0 { - log.Fatal("account balance is 0") - } - } - - log.Infof("account %s balance is %d", api.MyAddress().Hex(), acc.Balance) - - // Create a new census - censusID, err := api.NewCensus(vapi.CensusTypeZKWeighted) - if err != nil { - log.Fatal(err) - } - log.Infof("new census created with id %s", censusID.String()) - - // Generate n participant accounts - voterAccounts := ethereum.NewSignKeysBatch(c.nvotes) - - // Add the accounts to the census by batches - participants := &vapi.CensusParticipants{} - for i, voterAccount := range voterAccounts { - zkAddr, err := zk.AddressFromSignKeys(voterAccount) - if err != nil { - log.Fatal(err) - } - - participants.Participants = append(participants.Participants, - vapi.CensusParticipant{ - Key: zkAddr.Bytes(), - Weight: (*types.BigInt)(new(big.Int).SetUint64(10)), - }) - if i == len(voterAccounts)-1 || ((i+1)%vapi.MaxCensusAddBatchSize == 0) { - if err := api.CensusAddParticipants(censusID, participants); err != nil { - log.Fatal(err) - } - log.Infof("added %d participants to census %s", - len(participants.Participants), censusID.String()) - participants = &vapi.CensusParticipants{} - } - } - - // Check census size - size, err := api.CensusSize(censusID) - if err != nil { - log.Fatal(err) - } - if size != uint64(c.nvotes) { - log.Fatalf("census size is %d, expected %d", size, c.nvotes) - } - log.Infof("census %s size is %d", censusID.String(), size) - - // Publish the census - root, censusURI, err := api.CensusPublish(censusID) - if err != nil { - log.Fatal(err) - } - log.Infof("census published with root %s", root.String()) - - // Check census size (of the published census) - size, err = api.CensusSize(root) - if err != nil { - log.Fatal(err) - } - if size != uint64(c.nvotes) { - log.Fatalf("published census size is %d, expected %d", size, c.nvotes) - } - - // Create a new Election - electionID, err := api.NewElection(&vapi.ElectionDescription{ - Title: map[string]string{"default": fmt.Sprintf("Test election %s", util.RandomHex(8))}, - Description: map[string]string{"default": "Test election description"}, - EndDate: time.Now().Add(time.Minute * 20), - - VoteType: vapi.VoteType{ - UniqueChoices: false, - MaxVoteOverwrites: 1, - }, - - ElectionType: vapi.ElectionType{ - Autostart: true, - Interruptible: true, - Anonymous: true, - SecretUntilTheEnd: false, - DynamicCensus: false, - }, - - Census: vapi.CensusTypeDescription{ - RootHash: root, - URL: censusURI, - Type: vapi.CensusTypeZKWeighted, - Size: size, - }, - - Questions: []vapi.Question{ - { - Title: map[string]string{"default": "Test question 1"}, - Description: map[string]string{"default": "Test question 1 description"}, - Choices: []vapi.ChoiceMetadata{ - { - Title: map[string]string{"default": "Yes"}, - Value: 0, - }, - { - Title: map[string]string{"default": "No"}, - Value: 1, - }, - }, - }, - }, - }) - if err != nil { - log.Fatal(err) - } - - // Wait for the election creation - ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - election, err := api.WaitUntilElectionCreated(ctx, electionID) - if err != nil { - log.Errorw(err, "error creating the election") - return - } - log.Debugf("election details: %+v", *election) - log.Infof("created new election with id %s - now wait until it starts", electionID.String()) - - // Wait for the election to start - ctx, cancel = context.WithTimeout(context.Background(), time.Second*40) - defer cancel() - election, err = api.WaitUntilElectionStarts(ctx, electionID) - if err != nil { - log.Fatal(err) - } - - log.Debugf("election details: %+v", *election) - + // Send the votes (parallelized) startTime := time.Now() - successProofs, successVotes := 0, 0 - proofCh, voteCh, stopCh := make(chan bool), make(chan bool), make(chan bool) - go func() { - for { - select { - case <-proofCh: - successProofs++ - case <-voteCh: - successVotes++ - case <-stopCh: - return - } - } - }() + wg := sync.WaitGroup{} apiClientMtx := &sync.Mutex{} - addNaccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { + voteAccounts := func(accounts []*ethereum.SignKeys, wg *sync.WaitGroup) { defer wg.Done() - log.Infof("generating %d voting proofs", len(accounts)) + log.Infof("sending %d votes", len(accounts)) + // We use maps instead of slices to have the capacity of resending votes + // without repeating them. + accountsMap := make(map[int]*ethereum.SignKeys, len(accounts)) for i, acc := range accounts { - apiClientMtx.Lock() - - privKey := acc.PrivateKey() - if err = api.SetAccount(privKey.String()); err != nil { + accountsMap[i] = acc + } + // Send the votes + votesSent := 0 + for { + contextDeadlines := 0 + for i, voterAccount := range accountsMap { + apiClientMtx.Lock() + privKey := voterAccount.PrivateKey() + if err := api.SetAccount(privKey.String()); err != nil { + apiClientMtx.Unlock() + log.Fatal(err) + return + } + + _, err := api.Vote(&apiclient.VoteData{ + ElectionID: t.election.ElectionID, + ProofMkTree: t.proofs[voterAccount.Address().Hex()], + Choices: []int{i % 2}, + VotingWeight: new(big.Int).SetUint64(8), + }) apiClientMtx.Unlock() - log.Fatal(err) - return - } + // if the context deadline is reached, we don't need to print it (let's jus retry) + if err != nil && errors.Is(err, context.DeadlineExceeded) || os.IsTimeout(err) { + contextDeadlines++ + continue + } else if err != nil && !strings.Contains(err.Error(), "already exists") { + // if the error is not "vote already exists", we need to print it + log.Warn(err) + continue + } + // if the vote was sent successfully or already exists, we remove it from the accounts map + votesSent++ + delete(accountsMap, i) - pr, err := api.CensusGenProof(root, api.MyZkAddress().Bytes()) - if err != nil { - apiClientMtx.Unlock() - log.Warnw(err.Error(), - "current", i, - "total", c.nvotes) - continue } - - log.Debugw("census proof generated", - "current", i, - "total", len(accounts)) - proofCh <- true - - _, err = api.Vote(&apiclient.VoteData{ - ElectionID: electionID, - ProofMkTree: pr, - Choices: []int{i % 2}, - VotingWeight: new(big.Int).SetUint64(8), - }) - apiClientMtx.Unlock() - if err != nil && !strings.Contains(err.Error(), "already exists") { - // if the error is not "vote already exists", we need to print it - log.Warn(err.Error()) - continue + if len(accountsMap) == 0 { + break } - log.Debugw("vote sent", - "current", i, - "total", len(accounts)) - voteCh <- true - pr = nil + log.Infof("sent %d/%d votes... got %d HTTP errors", votesSent, len(accounts), contextDeadlines) + time.Sleep(time.Second * 5) } + log.Infof("successfully sent %d votes", votesSent) + time.Sleep(time.Second * 2) } pcount := c.nvotes / c.parallelCount - var wg sync.WaitGroup - for i := 0; i < len(voterAccounts); i += pcount { + for i := 0; i < len(t.voterAccounts); i += pcount { end := i + pcount - if end > len(voterAccounts) { - end = len(voterAccounts) + if end > len(t.voterAccounts) { + end = len(t.voterAccounts) } wg.Add(1) - go addNaccounts(voterAccounts[i:end], &wg) + go voteAccounts(t.voterAccounts[i:end], &wg) } wg.Wait() - time.Sleep(time.Second) // wait a grace time for the last proof to be added - log.Debugf("%d/%d voting proofs generated successfully", successProofs, len(voterAccounts)) - log.Debugf("%d/%d votes sent successfully", successVotes, len(voterAccounts)) - stopCh <- true + log.Infof("%d votes submitted successfully, took %s (%d votes/second)", + c.nvotes, time.Since(startTime), int(float64(c.nvotes)/time.Since(startTime).Seconds())) // Wait for all the votes to be verified log.Infof("waiting for all the votes to be registered...") for { - count, err := api.ElectionVoteCount(electionID) + count, err := api.ElectionVoteCount(t.election.ElectionID) if err != nil { log.Warn(err) } @@ -335,36 +157,36 @@ func (t *E2EAnonElection) Run() error { // End the election by setting the status to ENDED log.Infof("ending election...") - hash, err := api.SetElectionStatus(electionID, "ENDED") + hash, err := api.SetElectionStatus(t.election.ElectionID, "ENDED") if err != nil { log.Fatal(err) } // Check the election status is actually ENDED - ctx, cancel = context.WithTimeout(context.Background(), time.Second*40) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*40) defer cancel() if _, err := api.WaitUntilTxIsMined(ctx, hash); err != nil { log.Fatalf("gave up waiting for tx %s to be mined: %s", hash, err) } - election, err = api.Election(electionID) + t.election, err = api.Election(t.election.ElectionID) if err != nil { log.Fatal(err) } - if election.Status != "ENDED" { + if t.election.Status != "ENDED" { log.Fatal("election status is not ENDED") } - log.Infof("election %s status is ENDED", electionID.String()) + log.Infof("election %s status is ENDED", t.election.ElectionID.String()) // Wait for the election to be in RESULTS state ctx, cancel = context.WithTimeout(context.Background(), time.Second*300) defer cancel() - election, err = api.WaitUntilElectionStatus(ctx, electionID, "RESULTS") + t.election, err = api.WaitUntilElectionStatus(ctx, t.election.ElectionID, "RESULTS") if err != nil { log.Fatal(err) } - log.Infof("election %s status is RESULTS", electionID.String()) - log.Infof("election results: %v", election.Results) + log.Infof("election %s status is RESULTS", t.election.ElectionID.String()) + log.Infof("election results: %v", t.election.Results) return nil }