refine add/down node

Signed-off-by: husharp <[email protected]>

client/http/interface.go

@@ -49,6 +49,7 @@ type Client interface {
 	GetRegionStatusByKeyRange(context.Context, *KeyRange, bool) (*RegionStats, error)
 	GetStores(context.Context) (*StoresInfo, error)
 	GetStore(context.Context, uint64) (*StoreInfo, error)
+	DeleteStore(context.Context, uint64) error
 	SetStoreLabels(context.Context, int64, map[string]string) error
 	GetHealthStatus(context.Context) ([]Health, error)
 	/* Config-related interfaces */
@@ -440,6 +441,13 @@ func (c *client) GetStore(ctx context.Context, storeID uint64) (*StoreInfo, erro
 	return &store, nil
 }
 
+func (c *client) DeleteStore(ctx context.Context, storeID uint64) error {
+	return c.request(ctx, newRequestInfo().
+		WithName(deleteStoreName).
+		WithURI(StoreByID(storeID)).
+		WithMethod(http.MethodDelete))
+}
+
 // GetClusterVersion gets the cluster version.
 func (c *client) GetClusterVersion(ctx context.Context) (string, error) {
 	var version string

client/http/request_info.go

@@ -39,6 +39,7 @@ const (
 	getRegionStatusByKeyRangeName           = "GetRegionStatusByKeyRange"
 	getStoresName                           = "GetStores"
 	getStoreName                            = "GetStore"
+	deleteStoreName                         = "DeleteStore"
 	setStoreLabelsName                      = "SetStoreLabels"
 	getHealthStatusName                     = "GetHealthStatus"
 	getConfigName                           = "GetConfig"

conf/simconfig.toml

@@ -1,8 +1,10 @@
 # PD Simulator Configuration
 
-[tick]
 ## the tick interval when starting PD inside (default: "100ms")
 sim-tick-interval = "100ms"
+total-store = 10
+total-region = 10000
+case-name = "balance-leader"
 
 [store]
 ## the capacity size of a new store in GB (default: 1024)
@@ -11,8 +13,8 @@ store-capacity = 1024
 store-available = 1024
 ## the io rate of a new store in MB/s (default: 40)
 store-io-per-second = 40
-## the version of a new store (default: "2.1.0")
-store-version = "2.1.0"
+## the version of a new store (default: "8.1.0")
+store-version = "8.1.0"
 
 ## the meaning of these configurations below are similar with config.toml
 [server]

tools/pd-simulator/main.go

@@ -17,6 +17,7 @@ package main
 import (
 	"context"
 	"fmt"
+	pdHttp "github.com/tikv/pd/client/http"
 	"os"
 	"os/signal"
 	"syscall"
@@ -92,6 +93,7 @@ func main() {
 
 func run(simCase string, simConfig *sc.SimConfig) {
 	if *pdAddr != "" {
+		simulator.PDHTTPClient = pdHttp.NewClient("pd-simulator", []string{*pdAddr})
 		simStart(*pdAddr, *statusAddress, simCase, simConfig)
 	} else {
 		local, clean := NewSingleServer(context.Background(), simConfig)
@@ -105,6 +107,7 @@ func run(simCase string, simConfig *sc.SimConfig) {
 			}
 			time.Sleep(100 * time.Millisecond)
 		}
+		simulator.PDHTTPClient = pdHttp.NewClient("pd-simulator", []string{local.GetAddr()})
 		simStart(local.GetAddr(), "", simCase, simConfig, clean)
 	}
 }

tools/pd-simulator/simulator/cases/balance_leader.go

@@ -28,12 +28,15 @@ func newBalanceLeader(config *sc.SimConfig) *Case {
 
 	totalStore := config.TotalStore
 	totalRegion := config.TotalRegion
+	allStores := make(map[uint64]struct{}, totalStore)
 	replica := int(config.ServerConfig.Replication.MaxReplicas)
 	for i := 0; i < totalStore; i++ {
+		id := simutil.IDAllocator.NextID()
 		simCase.Stores = append(simCase.Stores, &Store{
-			ID:     simutil.IDAllocator.NextID(),
+			ID:     id,
 			Status: metapb.StoreState_Up,
 		})
+		allStores[id] = struct{}{}
 	}
 
 	leaderStoreID := simCase.Stores[totalStore-1].ID
@@ -58,10 +61,18 @@ func newBalanceLeader(config *sc.SimConfig) *Case {
 		})
 	}
 
-	simCase.Checker = func(regions *core.RegionsInfo, _ []info.StoreStats) bool {
-		for i := 1; i <= totalStore; i++ {
-			leaderCount := regions.GetStoreLeaderCount(uint64(i))
-			if !isUniform(leaderCount, totalRegion/totalStore) {
+	simCase.Checker = func(stores []*metapb.Store, regions *core.RegionsInfo, _ []info.StoreStats) bool {
+		for _, store := range stores {
+			if store.NodeState == metapb.NodeState_Removed {
+				delete(allStores, store.GetId())
+			}
+		}
+		if len(allStores) == 0 {
+			return false
+		}
+		for storeID := range allStores {
+			leaderCount := regions.GetStoreLeaderCount(storeID)
+			if !isUniform(leaderCount, totalRegion/len(allStores)) {
 				return false
 			}
 		}

tools/pd-simulator/simulator/cases/balance_region.go

@@ -30,6 +30,7 @@ func newRedundantBalanceRegion(config *sc.SimConfig) *Case {
 	totalStore := config.TotalStore
 	totalRegion := config.TotalRegion
 	replica := int(config.ServerConfig.Replication.MaxReplicas)
+	allStores := make(map[uint64]struct{}, totalStore)
 
 	for i := 0; i < totalStore; i++ {
 		s := &Store{
@@ -40,6 +41,7 @@ func newRedundantBalanceRegion(config *sc.SimConfig) *Case {
 			s.HasExtraUsedSpace = true
 		}
 		simCase.Stores = append(simCase.Stores, s)
+		allStores[s.ID] = struct{}{}
 	}
 
 	for i := 0; i < totalRegion; i++ {
@@ -57,21 +59,27 @@ func newRedundantBalanceRegion(config *sc.SimConfig) *Case {
 		})
 	}
 
-	storesLastUpdateTime := make([]int64, totalStore+1)
-	storeLastAvailable := make([]uint64, totalStore+1)
-	simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool {
+	storesLastUpdateTime := make(map[uint64]int64, totalStore)
+	storeLastAvailable := make(map[uint64]uint64, totalStore)
+	simCase.Checker = func(stores []*metapb.Store, regions *core.RegionsInfo, stats []info.StoreStats) bool {
+		for _, store := range stores {
+			if store.NodeState == metapb.NodeState_Removed {
+				delete(allStores, store.GetId())
+			}
+		}
+
 		curTime := time.Now().Unix()
-		for i := 1; i <= totalStore; i++ {
-			available := stats[i].GetAvailable()
-			if curTime-storesLastUpdateTime[i] > 60 {
-				if storeLastAvailable[i] != available {
+		for storeID := range allStores {
+			available := stats[storeID].GetAvailable()
+			if curTime-storesLastUpdateTime[storeID] > 60 {
+				if storeLastAvailable[storeID] != available {
 					return false
 				}
-				if stats[i].ToCompactionSize != 0 {
+				if stats[storeID].ToCompactionSize != 0 {
 					return false
 				}
-				storesLastUpdateTime[i] = curTime
-				storeLastAvailable[i] = available
+				storesLastUpdateTime[storeID] = curTime
+				storeLastAvailable[storeID] = available
 			} else {
 				return false
 			}

tools/pd-simulator/simulator/cases/cases.go

@@ -16,8 +16,8 @@ package cases
 
 import (
 	"github.com/pingcap/kvproto/pkg/metapb"
+	pdHttp "github.com/tikv/pd/client/http"
 	"github.com/tikv/pd/pkg/core"
-	"github.com/tikv/pd/pkg/schedule/placement"
 	"github.com/tikv/pd/pkg/utils/typeutil"
 	"github.com/tikv/pd/tools/pd-simulator/simulator/config"
 	"github.com/tikv/pd/tools/pd-simulator/simulator/info"
@@ -45,7 +45,7 @@ type Region struct {
 }
 
 // CheckerFunc checks if the scheduler is finished.
-type CheckerFunc func(*core.RegionsInfo, []info.StoreStats) bool
+type CheckerFunc func([]*metapb.Store, *core.RegionsInfo, []info.StoreStats) bool
 
 // Case represents a test suite for simulator.
 type Case struct {
@@ -57,7 +57,7 @@ type Case struct {
 	TableNumber     int
 
 	Checker CheckerFunc // To check the schedule is finished.
-	Rules   []*placement.Rule
+	Rules   []*pdHttp.Rule
 	Labels  typeutil.StringSlice
 }
 

tools/pd-simulator/simulator/cases/diagnose_label_isolation.go

@@ -33,18 +33,22 @@ func newLabelNotMatch1(_ *sc.SimConfig) *Case {
 
 	num1, num2 := 3, 1
 	storeNum, regionNum := num1+num2, 200
+	allStores := make(map[uint64]struct{}, storeNum+1)
 	for i := 0; i < num1; i++ {
 		id := IDAllocator.nextID()
 		simCase.Stores = append(simCase.Stores, &Store{
 			ID:     id,
 			Status: metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{{Key: "host", Value: fmt.Sprintf("host%d", id)}},
 		})
+		allStores[id] = struct{}{}
 	}
+	id := IDAllocator.nextID()
 	simCase.Stores = append(simCase.Stores, &Store{
-		ID:     IDAllocator.nextID(),
+		ID:     id,
 		Status: metapb.StoreState_Up,
 	})
+	allStores[id] = struct{}{}
 
 	for i := 0; i < regionNum; i++ {
 		peers := []*metapb.Peer{
@@ -61,24 +65,30 @@ func newLabelNotMatch1(_ *sc.SimConfig) *Case {
 		})
 	}
 
-	storesLastUpdateTime := make([]int64, storeNum+1)
-	storeLastAvailable := make([]uint64, storeNum+1)
-	simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool {
+	storesLastUpdateTime := make(map[uint64]int64, storeNum+1)
+	storeLastAvailable := make(map[uint64]uint64, storeNum+1)
+	simCase.Checker = func(stores []*metapb.Store, regions *core.RegionsInfo, stats []info.StoreStats) bool {
+		for _, store := range stores {
+			if store.NodeState == metapb.NodeState_Removed {
+				delete(allStores, store.GetId())
+			}
+		}
+
 		res := true
 		curTime := time.Now().Unix()
 		storesAvailable := make([]uint64, 0, storeNum+1)
-		for i := 1; i <= storeNum; i++ {
-			available := stats[i].GetAvailable()
+		for storeID := range allStores {
+			available := stats[storeID].GetAvailable()
 			storesAvailable = append(storesAvailable, available)
-			if curTime-storesLastUpdateTime[i] > 360 {
-				if storeLastAvailable[i] != available {
+			if curTime-storesLastUpdateTime[storeID] > 360 {
+				if storeLastAvailable[storeID] != available {
 					res = false
 				}
-				if stats[i].ToCompactionSize != 0 {
+				if stats[storeID].ToCompactionSize != 0 {
 					res = false
 				}
-				storesLastUpdateTime[i] = curTime
-				storeLastAvailable[i] = available
+				storesLastUpdateTime[storeID] = curTime
+				storeLastAvailable[storeID] = available
 			} else {
 				res = false
 			}
@@ -95,21 +105,24 @@ func newLabelIsolation1(_ *sc.SimConfig) *Case {
 
 	num1, num2 := 2, 2
 	storeNum, regionNum := num1+num2, 300
+	allStores := make(map[uint64]struct{}, storeNum+1)
 	for i := 0; i < num1; i++ {
 		id := IDAllocator.nextID()
 		simCase.Stores = append(simCase.Stores, &Store{
 			ID:     id,
 			Status: metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{{Key: "host", Value: fmt.Sprintf("host%d", id)}},
 		})
+		allStores[id] = struct{}{}
 	}
-	id := IDAllocator.GetID() + 1
 	for i := 0; i < num2; i++ {
+		id := IDAllocator.nextID()
 		simCase.Stores = append(simCase.Stores, &Store{
-			ID:     IDAllocator.nextID(),
+			ID:     id,
 			Status: metapb.StoreState_Up,
 			Labels: []*metapb.StoreLabel{{Key: "host", Value: fmt.Sprintf("host%d", id)}},
 		})
+		allStores[id] = struct{}{}
 	}
 
 	for i := 0; i < regionNum; i++ {
@@ -127,24 +140,30 @@ func newLabelIsolation1(_ *sc.SimConfig) *Case {
 		})
 	}
 
-	storesLastUpdateTime := make([]int64, storeNum+1)
-	storeLastAvailable := make([]uint64, storeNum+1)
-	simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool {
+	storesLastUpdateTime := make(map[uint64]int64, storeNum)
+	storeLastAvailable := make(map[uint64]uint64, storeNum)
+	simCase.Checker = func(stores []*metapb.Store, regions *core.RegionsInfo, stats []info.StoreStats) bool {
+		for _, store := range stores {
+			if store.NodeState == metapb.NodeState_Removed {
+				delete(allStores, store.GetId())
+			}
+		}
+
 		res := true
 		curTime := time.Now().Unix()
 		storesAvailable := make([]uint64, 0, storeNum+1)
-		for i := 1; i <= storeNum; i++ {
-			available := stats[i].GetAvailable()
+		for storeID := range allStores {
+			available := stats[storeID].GetAvailable()
 			storesAvailable = append(storesAvailable, available)
-			if curTime-storesLastUpdateTime[i] > 360 {
-				if storeLastAvailable[i] != available {
+			if curTime-storesLastUpdateTime[storeID] > 360 {
+				if storeLastAvailable[storeID] != available {
 					res = false
 				}
-				if stats[i].ToCompactionSize != 0 {
+				if stats[storeID].ToCompactionSize != 0 {
 					res = false
 				}
-				storesLastUpdateTime[i] = curTime
-				storeLastAvailable[i] = available
+				storesLastUpdateTime[storeID] = curTime
+				storeLastAvailable[storeID] = available
 			} else {
 				res = false
 			}
@@ -160,12 +179,14 @@ func newLabelIsolation2(_ *sc.SimConfig) *Case {
 	simCase.Labels = []string{"dc", "zone", "host"}
 
 	storeNum, regionNum := 5, 200
+	allStores := make(map[uint64]struct{}, storeNum)
 	for i := 0; i < storeNum; i++ {
 		id := IDAllocator.nextID()
 		simCase.Stores = append(simCase.Stores, &Store{
 			ID:     id,
 			Status: metapb.StoreState_Up,
 		})
+		allStores[id] = struct{}{}
 	}
 	simCase.Stores[0].Labels = []*metapb.StoreLabel{{Key: "dc", Value: "dc1"}, {Key: "zone", Value: "zone1"}, {Key: "host", Value: "host1"}}
 	simCase.Stores[1].Labels = []*metapb.StoreLabel{{Key: "dc", Value: "dc1"}, {Key: "zone", Value: "zone1"}, {Key: "host", Value: "host2"}}
@@ -188,24 +209,30 @@ func newLabelIsolation2(_ *sc.SimConfig) *Case {
 		})
 	}
 
-	storesLastUpdateTime := make([]int64, storeNum+1)
-	storeLastAvailable := make([]uint64, storeNum+1)
-	simCase.Checker = func(_ *core.RegionsInfo, stats []info.StoreStats) bool {
+	storesLastUpdateTime := make(map[uint64]int64, storeNum)
+	storeLastAvailable := make(map[uint64]uint64, storeNum)
+	simCase.Checker = func(stores []*metapb.Store, regions *core.RegionsInfo, stats []info.StoreStats) bool {
+		for _, store := range stores {
+			if store.NodeState == metapb.NodeState_Removed {
+				delete(allStores, store.GetId())
+			}
+		}
+
 		res := true
 		curTime := time.Now().Unix()
 		storesAvailable := make([]uint64, 0, storeNum+1)
-		for i := 1; i <= storeNum; i++ {
-			available := stats[i].GetAvailable()
+		for storeID := range allStores {
+			available := stats[storeID].GetAvailable()
 			storesAvailable = append(storesAvailable, available)
-			if curTime-storesLastUpdateTime[i] > 360 {
-				if storeLastAvailable[i] != available {
+			if curTime-storesLastUpdateTime[storeID] > 360 {
+				if storeLastAvailable[storeID] != available {
 					res = false
 				}
-				if stats[i].ToCompactionSize != 0 {
+				if stats[storeID].ToCompactionSize != 0 {
 					res = false
 				}
-				storesLastUpdateTime[i] = curTime
-				storeLastAvailable[i] = available
+				storesLastUpdateTime[storeID] = curTime
+				storeLastAvailable[storeID] = available
 			} else {
 				res = false
 			}