Merge pull request #32 from irisnet/develop

R4R: merge to master

Dockerfile

@@ -0,0 +1,24 @@
+FROM golang:1.13.1-alpine3.10 as builder
+
+# Set up dependencies
+ENV PACKAGES go make git libc-dev bash
+
+# Set up path
+ENV REPO_PATH	$GOPATH/src
+ENV GO111MODULE on
+
+# RUN mkdir -p $REPO_PATH
+
+COPY . $REPO_PATH
+WORKDIR $REPO_PATH
+
+# Install minimum necessary dependencies, build binary
+RUN apk add --no-cache $PACKAGES && \
+    cd $REPO_PATH && make all
+
+FROM alpine:3.10
+
+ENV BINARY_NAME	rainbow-sync-iris
+COPY --from=builder /go/src/$BINARY_NAME /usr/local/bin/$BINARY_NAME
+
+CMD $BINARY_NAME

service/iris/Makefile → Makefile

@@ -3,15 +3,16 @@ GOBUILD=$(GOCMD) build
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test
 GOGET=$(GOCMD) get
-BINARY_NAME=rainbow-sync
+GOMOD=$(GOCMD) mod
+BINARY_NAME=rainbow-sync-iris
 BINARY_UNIX=$(BINARY_NAME)-unix
 
-all: get_tools get_deps build
+all: get_vendor build
 
-get_deps:
+get_vendor:
 	@rm -rf vendor/
-	@echo "--> Running dep ensure"
-	@dep ensure -v
+	@echo "--> Running go mod vendor"
+	$(GOMOD) vendor
 
 build:
 	$(GOBUILD) -o $(BINARY_NAME) -v
@@ -30,14 +31,4 @@ run:
 build-linux:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BINARY_UNIX) -v
 
-######################################
-## Tools
-
-check_tools:
-	cd tools && $(MAKE) check_tools
-
-get_tools:
-	cd tools && $(MAKE) get_tools
-
-update_tools:
-	cd tools && $(MAKE) update_tools
+######################################

README.md

@@ -1,25 +1,33 @@
 # rainbow-sync
 A daemon that synchronizes IRIS hub data for the Rainbow wallet backend
 
+# Structure
 
-## Run
-- Cosmos
-```bash
-cd service/cosmos && make all
-nohup ./rainbow-sync > debug.log 2>&1 &
-```
-- Iris
-```bash
-cd service/iris && make all
-nohup ./rainbow-sync > debug.log 2>&1 &
-```
+- `conf`: config of project
+- `block`: parse asset detail and tx function module
+- `model`: mongodb script to create database
+- `task`: main logic of sync-server, sync data from blockChain and write to database
+- `db`: database model
+- `helper`: helper functions
+- `utils`: common functions
+- `main.go`: bootstrap project
+
+# SetUp
+## Database
+Use Mongodb  to store IRIS hub data
+
+# Build And Run
+
+- Build: `make all`
+- Run: `make run`
+- Cross compilation: `make build-linux`
 
 ## Run with docker
 You can run application with docker.
-### Iris
+### Image
 - Build Rainbow-sync Image
 ```$xslt
-cd service/iris && docker build -t rainbow-sync:dev01 .
+docker build -t rainbow-sync .
 ```
 
 - Run Application
@@ -29,24 +37,11 @@ docker run --name rainbow-sync \&
 -e "DB_ADDR=127.0.0.1:27217" -e "DB_USER=user" \&
 -e "DB_PASSWD=password" -e "DB_DATABASE=db_name" \&
 -e "IRIS_NETWORK=testnet" \&
--e "SER_BC_FULL_NODE=tcp://localhost:26657,..." rainbow-sync:dev01
-```
-### Cosmos
-- Build Rainbow-sync Image
-```$xslt
-cd service/cosmos && docker build -t rainbow-sync:dev01 .
-```
-- Run Application
-```$xslt
-docker run --name rainbow-sync \&
--v /mnt/data/rainbow-sync/logs:/root/go/src/github.com/irisnet/rainbow-sync/logs \&
--e "DB_ADDR=127.0.0.1:27217" -e "DB_USER=user" \&
--e "DB_PASSWD=password" -e "DB_DATABASE=db_name" \&
--e "SER_BC_FULL_NODE_COSMOS=tcp://localhost:36657,..."  rainbow-sync:dev01
+-e "SER_BC_FULL_NODES=tcp://localhost:26657,..." rainbow-sync
 ```
 
 
-## environment params
+## Environment Params
 
 | param | type | default |description | example |
 | :--- | :--- | :--- | :---: | :---: |
@@ -55,15 +50,16 @@ docker run --name rainbow-sync \&
 | DB_PASSWD | string | "" |db passwd  | password |
 | DB_DATABASE | string | "" |database name  | db_name |
 | IRIS_NETWORK | string | "testnet" |irishub name  | testnet or mainnet |
-| SER_BC_FULL_NODE | string | tcp://localhost:26657 | iris full node rpc url | tcp://localhost:26657, tcp://127.0.0.2:26657 |
-| WORKER_NUM_CREATE_TASK | string | 2 | 创建同步Iris的Tag任务的线程数 | 2 |
-| WORKER_NUM_EXECUTE_TASK | string | 30 | 执行同步Iris的Tag任务的线程数 | 30 |
-| WORKER_MAX_SLEEP_TIME | string | 120 | 允许同步Iris的Tag线程处于不工作状态的最大时长（单位为：秒） | 120 |
-| BLOCK_NUM_PER_WORKER_HANDLE | string | 50 | 每个同步Iris的Tag任务所包含的Iris区块数 | 50 |
-| SER_BC_FULL_NODE_COSMOS | string | tcp://localhost:36657 |cosmos full node rpc url  | tcp://localhost:36657, tcp://127.0.0.2:36657 |
-| WORKER_NUM_CREATE_TASK_COSMOS | string | 2 | 创建同步Cosmos的Tx任务的线程数 | 2 |
-| WORKER_NUM_EXECUTE_TASK_COSMOS | string | 30 | 执行同步Cosmos的Tx任务的线程数 | 30 |
-| WORKER_MAX_SLEEP_TIME_COSMOS | string | 120 | 允许同步Cosmos的Tx线程处于不工作状态的最大时长（单位为：秒） | 120 |
-| BLOCK_NUM_PER_WORKER_HANDLE_COSMOS | string | 50 | 每个同步Cosmos的Tx任务所包含的Cosmos区块数 | 50 |
-
+| SER_BC_FULL_NODES | string | tcp://localhost:26657 | iris full node rpc url | tcp://localhost:26657, tcp://127.0.0.2:26657 |
+| WORKER_NUM_EXECUTE_TASK | string | 30 | number of threads executing synchronization TX task | 30 |
+| WORKER_MAX_SLEEP_TIME | string | 120 | the maximum time (in seconds) that synchronization TX threads are allowed to be out of work | 120 |
+| BLOCK_NUM_PER_WORKER_HANDLE | string | 50 | number of blocks per sync TX task | 50 |
 
+- Remarks
+  - synchronizes  irishub data from  specify block height(such as:17908 current time:1576208532)
+
+     At first,stop the rainbow-sync and create the task. Run:
+  ```bash
+     db.sync_iris_task.insert({'start_height':NumberLong(17908),'end_height':NumberLong(0),'current_height':NumberLong(0),'status':'unhandled','last_update_time':NumberLong(1576208532)})
+  ```
+  Then,start the rainbow-sync.

service/iris/block/parse_asset_detail.go → block/parse_asset_detail.go

@@ -1,10 +1,10 @@
 package block
 
 import (
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
-	model "github.com/irisnet/rainbow-sync/service/iris/db"
-	imodel "github.com/irisnet/rainbow-sync/service/iris/model"
-	"github.com/irisnet/rainbow-sync/service/iris/helper"
+	"github.com/irisnet/rainbow-sync/logger"
+	model "github.com/irisnet/rainbow-sync/db"
+	imodel "github.com/irisnet/rainbow-sync/model"
+	"github.com/irisnet/rainbow-sync/helper"
 	"strings"
 	"gopkg.in/mgo.v2/txn"
 	"gopkg.in/mgo.v2/bson"

service/iris/block/parse_asset_detail_test.go → block/parse_asset_detail_test.go

@@ -2,10 +2,10 @@ package block
 
 import (
 	"testing"
-	irisConf "github.com/irisnet/rainbow-sync/service/iris/conf"
+	irisConf "github.com/irisnet/rainbow-sync/conf"
 	"encoding/json"
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
-	"github.com/irisnet/rainbow-sync/service/iris/helper"
+	"github.com/irisnet/rainbow-sync/logger"
+	"github.com/irisnet/rainbow-sync/helper"
 )
 
 func TestIris_Block_ParseIrisAssetDetail(t *testing.T) {

service/iris/block/parse_tx.go → block/parse_tx.go

@@ -1,15 +1,15 @@
 package block
 
 import (
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
-	imodel "github.com/irisnet/rainbow-sync/service/iris/model"
-	imsg "github.com/irisnet/rainbow-sync/service/iris/model/msg"
-	"github.com/irisnet/rainbow-sync/service/iris/utils"
+	"github.com/irisnet/rainbow-sync/logger"
+	imodel "github.com/irisnet/rainbow-sync/model"
+	imsg "github.com/irisnet/rainbow-sync/model/msg"
+	"github.com/irisnet/rainbow-sync/utils"
 	abci "github.com/tendermint/tendermint/abci/types"
 	"github.com/irisnet/irishub/app/v1/auth"
 	"github.com/tendermint/tendermint/types"
-	"github.com/irisnet/rainbow-sync/service/iris/helper"
-	"github.com/irisnet/rainbow-sync/service/iris/constant"
+	"github.com/irisnet/rainbow-sync/helper"
+	"github.com/irisnet/rainbow-sync/constant"
 )
 
 // parse iris txs  from block result txs

service/iris/block/parse_tx_test.go → block/parse_tx_test.go

@@ -2,8 +2,8 @@ package block
 
 import (
 	"testing"
-	"github.com/irisnet/rainbow-sync/service/iris/helper"
-	irisConf "github.com/irisnet/rainbow-sync/service/iris/conf"
+	"github.com/irisnet/rainbow-sync/helper"
+	irisConf "github.com/irisnet/rainbow-sync/conf"
 	"encoding/json"
 )
 

service/iris/conf/db/types.go → conf/db/types.go

@@ -1,8 +1,8 @@
 package db
 
 import (
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
-	constant "github.com/irisnet/rainbow-sync/service/iris/conf"
+	"github.com/irisnet/rainbow-sync/logger"
+	constant "github.com/irisnet/rainbow-sync/conf"
 	"os"
 )
 

service/iris/conf/types.go → conf/types.go

@@ -1,7 +1,7 @@
 package conf
 
 import (
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
+	"github.com/irisnet/rainbow-sync/logger"
 	"os"
 	"strconv"
 	"strings"
@@ -11,7 +11,7 @@ var (
 	BlockChainMonitorUrl = []string{"tcp://192.168.150.31:26657"}
 
 	IrisNetwork             = "testnet"
-	WorkerNumCreateTask     = 2
+	WorkerNumCreateTask     = 1
 	WorkerNumExecuteTask    = 30
 	WorkerMaxSleepTime      = 2 * 60
 	BlockNumPerWorkerHandle = 50
@@ -26,8 +26,7 @@ const (
 	EnvNameDbPassWd   = "DB_PASSWD"
 	EnvNameDbDataBase = "DB_DATABASE"
 
-	EnvNameSerNetworkFullNode      = "SER_BC_FULL_NODE"
-	EnvNameWorkerNumCreateTask     = "WORKER_NUM_CREATE_TASK"
+	EnvNameSerNetworkFullNodes     = "SER_BC_FULL_NODES"
 	EnvNameWorkerNumExecuteTask    = "WORKER_NUM_EXECUTE_TASK"
 	EnvNameWorkerMaxSleepTime      = "WORKER_MAX_SLEEP_TIME"
 	EnvNameBlockNumPerWorkerHandle = "BLOCK_NUM_PER_WORKER_HANDLE"
@@ -38,26 +37,18 @@ const (
 func init() {
 	var err error
 
-	nodeUrl, found := os.LookupEnv(EnvNameSerNetworkFullNode)
+	nodeUrl, found := os.LookupEnv(EnvNameSerNetworkFullNodes)
 	if found {
 		BlockChainMonitorUrl = strings.Split(nodeUrl, ",")
 	}
-	logger.Info("Env Value", logger.Any(EnvNameSerNetworkFullNode, BlockChainMonitorUrl))
+	logger.Info("Env Value", logger.Any(EnvNameSerNetworkFullNodes, BlockChainMonitorUrl))
 
-	workerNumCreateTask, found := os.LookupEnv(EnvNameWorkerNumCreateTask)
-	if found {
-		WorkerNumCreateTask, err = strconv.Atoi(workerNumCreateTask)
-		if err != nil {
-			logger.Fatal("Can't convert str to int", logger.String(EnvNameWorkerNumCreateTask, workerNumCreateTask))
-		}
-	}
-	logger.Info("Env Value", logger.Int(EnvNameWorkerNumCreateTask, WorkerNumCreateTask))
 
 	workerNumExecuteTask, found := os.LookupEnv(EnvNameWorkerNumExecuteTask)
 	if found {
 		WorkerNumExecuteTask, err = strconv.Atoi(workerNumExecuteTask)
 		if err != nil {
-			logger.Fatal("Can't convert str to int", logger.String(EnvNameWorkerNumExecuteTask, workerNumCreateTask))
+			logger.Fatal("Can't convert str to int", logger.String(EnvNameWorkerNumExecuteTask, workerNumExecuteTask))
 		}
 	}
 	logger.Info("Env Value", logger.Int(EnvNameWorkerNumExecuteTask, WorkerNumExecuteTask))

service/iris/cron/cron_task.go → cron/cron_task.go

@@ -4,11 +4,11 @@ import (
 	"time"
 	"os"
 	"os/signal"
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
-	"github.com/irisnet/rainbow-sync/service/iris/db"
-	model "github.com/irisnet/rainbow-sync/service/iris/model"
-	"github.com/irisnet/rainbow-sync/service/iris/block"
-	"github.com/irisnet/rainbow-sync/service/iris/helper"
+	"github.com/irisnet/rainbow-sync/logger"
+	"github.com/irisnet/rainbow-sync/db"
+	model "github.com/irisnet/rainbow-sync/model"
+	"github.com/irisnet/rainbow-sync/block"
+	"github.com/irisnet/rainbow-sync/helper"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 	"gopkg.in/mgo.v2/txn"

service/iris/cron/cron_task_test.go → cron/cron_task_test.go

@@ -3,9 +3,9 @@ package cron
 import (
 	"testing"
 	"time"
-	"github.com/irisnet/rainbow-sync/service/iris/db"
-	"github.com/irisnet/rainbow-sync/service/iris/helper"
-	"github.com/irisnet/rainbow-sync/service/iris/conf"
+	"github.com/irisnet/rainbow-sync/db"
+	"github.com/irisnet/rainbow-sync/helper"
+	"github.com/irisnet/rainbow-sync/conf"
 )
 
 func TestGetUnknownTxsByPage(t *testing.T) {

service/iris/db/db.go → db/db.go

@@ -2,8 +2,8 @@ package db
 
 import (
 	"fmt"
-	conf "github.com/irisnet/rainbow-sync/service/iris/conf/db"
-	"github.com/irisnet/rainbow-sync/service/iris/logger"
+	conf "github.com/irisnet/rainbow-sync/conf/db"
+	"github.com/irisnet/rainbow-sync/logger"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 	"gopkg.in/mgo.v2/txn"
@@ -46,6 +46,20 @@ func getSession() *mgo.Session {
 	return session.Clone()
 }
 
+func EnsureIndexes(collectionName string, indexes []mgo.Index) {
+	session := getSession()
+	defer session.Close()
+	c := session.DB(conf.Database).C(collectionName)
+	if len(indexes) > 0 {
+		for _, v := range indexes {
+			if err := c.EnsureIndex(v); err != nil {
+				logger.Warn("ensure index fail", logger.String("collectionName", collectionName),
+					logger.String("err", err.Error()))
+			}
+		}
+	}
+}
+
 // get collection object
 func ExecCollection(collectionName string, s func(*mgo.Collection) error) error {
 	session := getSession()

service/iris/db/types.go → db/types.go

@@ -12,5 +12,7 @@ type (
 		Name() string
 		// primary key pair(used to find a unique record)
 		PkKvPair() map[string]interface{}
+		// ensure indexes
+		EnsureIndexes()
 	}
 )

go.mod

@@ -0,0 +1,19 @@
+module github.com/irisnet/rainbow-sync
+
+go 1.13
+
+require (
+	github.com/irisnet/irishub v0.16.0
+	github.com/jolestar/go-commons-pool v2.0.0+incompatible
+	github.com/tendermint/tendermint v0.32.8
+	go.uber.org/zap v1.13.0
+	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect
+)
+
+replace (
+	github.com/tendermint/iavl => github.com/irisnet/iavl v0.12.3
+	github.com/tendermint/tendermint => github.com/irisnet/tendermint v0.32.1
+	golang.org/x/crypto => github.com/tendermint/crypto v0.0.0-20180820045704-3764759f34a5
+)