persist region leader

Signed-off-by: Ryan Leung <[email protected]>

client/go.mod

@@ -42,3 +42,5 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/pingcap/kvproto => github.com/rleungx/kvproto v0.0.0-20240603031249-71f0a71d7c44

client/go.sum

@@ -46,8 +46,6 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm
 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg=
 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0=
 github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew=
-github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2 h1:364A6VCS+l0oHBKZKotX9LzmfEtIO/NTccTIQcPp3Ug=
-github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw=
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -63,6 +61,8 @@ github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqSc
 github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/rleungx/kvproto v0.0.0-20240603031249-71f0a71d7c44 h1:yX9wefuxilXswyCTIKeB7kPdQqb3umeHpzYx/6unl9I=
+github.com/rleungx/kvproto v0.0.0-20240603031249-71f0a71d7c44/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=

go.mod

@@ -113,7 +113,6 @@ require (
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20211122183932-1daafda22083 // indirect
@@ -200,3 +199,5 @@ require (
 	moul.io/zapgorm2 v1.1.0 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )
+
+replace github.com/pingcap/kvproto => github.com/rleungx/kvproto v0.0.0-20240603031249-71f0a71d7c44

pkg/core/region.go

@@ -61,7 +61,6 @@ type RegionInfo struct {
 	learners          []*metapb.Peer
 	witnesses         []*metapb.Peer
 	voters            []*metapb.Peer
-	leader            *metapb.Peer
 	downPeers         []*pdpb.PeerStats
 	pendingPeers      []*metapb.Peer
 	cpuUsage          uint64
@@ -123,10 +122,9 @@ func (r *RegionInfo) GetRef() int32 {
 }
 
 // NewRegionInfo creates RegionInfo with region's meta and leader peer.
-func NewRegionInfo(region *metapb.Region, leader *metapb.Peer, opts ...RegionCreateOption) *RegionInfo {
+func NewRegionInfo(region *metapb.Region, opts ...RegionCreateOption) *RegionInfo {
 	regionInfo := &RegionInfo{
-		meta:   region,
-		leader: leader,
+		meta: region,
 	}
 	for _, opt := range opts {
 		opt(regionInfo)
@@ -162,7 +160,7 @@ func classifyVoterAndLearner(region *RegionInfo) {
 // peersEqualTo returns true when the peers are not changed, which may caused by: the region leader not changed,
 // peer transferred, new peer was created, learners changed, pendingPeers changed.
 func (r *RegionInfo) peersEqualTo(region *RegionInfo) bool {
-	return r.leader.GetId() == region.leader.GetId() &&
+	return r.GetLeader().GetId() == region.GetLeader().GetId() &&
 		SortedPeersEqual(r.GetVoters(), region.GetVoters()) &&
 		SortedPeersEqual(r.GetLearners(), region.GetLearners()) &&
 		SortedPeersEqual(r.GetWitnesses(), region.GetWitnesses()) &&
@@ -226,7 +224,6 @@ func RegionFromHeartbeat(heartbeat RegionHeartbeatRequest, opts ...RegionCreateO
 	region := &RegionInfo{
 		term:            heartbeat.GetTerm(),
 		meta:            heartbeat.GetRegion(),
-		leader:          heartbeat.GetLeader(),
 		downPeers:       heartbeat.GetDownPeers(),
 		pendingPeers:    heartbeat.GetPendingPeers(),
 		writtenBytes:    heartbeat.GetBytesWritten(),
@@ -239,6 +236,7 @@ func RegionFromHeartbeat(heartbeat RegionHeartbeatRequest, opts ...RegionCreateO
 		queryStats:      heartbeat.GetQueryStats(),
 		source:          Heartbeat,
 	}
+	region.meta.Leader = heartbeat.GetLeader()
 
 	// scheduling service doesn't need the following fields.
 	if h, ok := heartbeat.(*pdpb.RegionHeartbeatRequest); ok {
@@ -298,7 +296,6 @@ func (r *RegionInfo) Clone(opts ...RegionCreateOption) *RegionInfo {
 	region := &RegionInfo{
 		term:              r.term,
 		meta:              typeutil.DeepClone(r.meta, RegionFactory),
-		leader:            typeutil.DeepClone(r.leader, RegionPeerFactory),
 		downPeers:         downPeers,
 		pendingPeers:      pendingPeers,
 		cpuUsage:          r.cpuUsage,
@@ -477,7 +474,7 @@ func (r *RegionInfo) GetFollowers() map[uint64]*metapb.Peer {
 	peers := r.GetVoters()
 	followers := make(map[uint64]*metapb.Peer, len(peers))
 	for _, peer := range peers {
-		if r.leader == nil || r.leader.GetId() != peer.GetId() {
+		if r.GetLeader() == nil || r.GetLeader().GetId() != peer.GetId() {
 			followers[peer.GetStoreId()] = peer
 		}
 	}
@@ -487,7 +484,7 @@ func (r *RegionInfo) GetFollowers() map[uint64]*metapb.Peer {
 // GetFollower randomly returns a follow peer.
 func (r *RegionInfo) GetFollower() *metapb.Peer {
 	for _, peer := range r.GetVoters() {
-		if r.leader == nil || r.leader.GetId() != peer.GetId() {
+		if r.GetLeader() == nil || r.GetLeader().GetId() != peer.GetId() {
 			return peer
 		}
 	}
@@ -680,7 +677,7 @@ func (r *RegionInfo) GetWriteRate() (bytesRate, keysRate float64) {
 
 // GetLeader returns the leader of the region.
 func (r *RegionInfo) GetLeader() *metapb.Peer {
-	return r.leader
+	return r.meta.Leader
 }
 
 // GetStartKey returns the start key of the region.
@@ -812,7 +809,7 @@ func GenerateRegionGuideFunc(enableLog bool) RegionGuideFunc {
 					)
 				}
 				// We check it first and do not return because the log is important for us to investigate,
-				saveCache, needSync = true, true
+				saveKV, saveCache, needSync = true, true, true
 			}
 			if len(region.GetPeers()) != len(origin.GetPeers()) {
 				saveKV, saveCache = true, true
@@ -1147,7 +1144,7 @@ func (r *RegionsInfo) updateSubTreeLocked(rangeChanged bool, overlaps []*RegionI
 	}
 	for _, peer := range region.GetVoters() {
 		storeID := peer.GetStoreId()
-		if peer.GetId() == region.leader.GetId() {
+		if peer.GetId() == region.GetLeader().GetId() {
 			setPeer(r.leaders, storeID)
 		} else {
 			setPeer(r.followers, storeID)
@@ -1286,7 +1283,7 @@ func (r *RegionsInfo) updateSubTreeStat(origin *RegionInfo, region *RegionInfo)
 	}
 	for _, peer := range region.GetVoters() {
 		storeID := peer.GetStoreId()
-		if peer.GetId() == region.leader.GetId() {
+		if peer.GetId() == region.GetLeader().GetId() {
 			updatePeerStat(r.leaders, storeID)
 		} else {
 			updatePeerStat(r.followers, storeID)
@@ -2170,7 +2167,7 @@ func MergeRegions(regions []*RegionInfo) []*RegionInfo {
 			region.meta.RegionEpoch = right.GetRegionEpoch()
 		}
 		region.meta.RegionEpoch.Version++
-		region.leader = left.leader
+		region.meta.Leader = left.GetLeader()
 		results = append(results, region)
 	}
 	return results
@@ -2188,6 +2185,7 @@ func NewTestRegionInfo(regionID, storeID uint64, start, end []byte, opts ...Regi
 		EndKey:      end,
 		Peers:       []*metapb.Peer{leader},
 		RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
+		Leader:      leader,
 	}
-	return NewRegionInfo(metaRegion, leader, opts...)
+	return NewRegionInfo(metaRegion, opts...)
 }

pkg/core/region_option.go

@@ -96,7 +96,7 @@ func WithLearners(learners []*metapb.Peer) RegionCreateOption {
 // WithLeader sets the leader for the region.
 func WithLeader(leader *metapb.Peer) RegionCreateOption {
 	return func(region *RegionInfo) {
-		region.leader = leader
+		region.meta.Leader = leader
 	}
 }