Skip to content

Commit

Permalink
core:fix style of hot history region info of startkey and endkey,fix …
Browse files Browse the repository at this point in the history 
…bug isleader always false (#4450)

bugfix:change the style of data store in levledb
bugfix:remove omit empty

ref #4449

Signed-off-by: qidi1 <[email protected]>

Co-authored-by: Ti Chi Robot <[email protected]>
  • Loading branch information
@qidi1 @ti-chi-bot
qidi1 and ti-chi-bot authored Dec 20, 2021
1 parent 7245220 commit af174e6
Show file tree
Hide file tree
Showing 4 changed files with 47 additions and 31 deletions.
2 changes: 1 addition & 1 deletion conf/config.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@
## Controls the time interval between write hot regions info into leveldb
# hot-regions-write-interval= "10m"
## The day of hot regions data to be reserved. 0 means close.
# hot-regions-reserved-days= "7"
# hot-regions-reserved-days= 7
## The number of Leader scheduling tasks performed at the same time.
# leader-schedule-limit = 4
## The number of Region scheduling tasks performed at the same time.
Expand Down
43 changes: 21 additions & 22 deletions server/core/hot_region_storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,19 +62,19 @@ type HistoryHotRegions struct {
// HistoryHotRegion wraps hot region info
// it is storage format of hot_region_storage
type HistoryHotRegion struct {
UpdateTime int64 `json:"update_time,omitempty"`
RegionID uint64 `json:"region_id,omitempty"`
PeerID uint64 `json:"peer_id,omitempty"`
StoreID uint64 `json:"store_id,omitempty"`
IsLeader bool `json:"is_leader,omitempty"`
IsLearner bool `json:"is_learner,omitempty"`
HotRegionType string `json:"hot_region_type,omitempty"`
HotDegree int64 `json:"hot_degree,omitempty"`
FlowBytes float64 `json:"flow_bytes,omitempty"`
KeyRate float64 `json:"key_rate,omitempty"`
QueryRate float64 `json:"query_rate,omitempty"`
StartKey []byte `json:"start_key,omitempty"`
EndKey []byte `json:"end_key,omitempty"`
UpdateTime int64 `json:"update_time"`
RegionID uint64 `json:"region_id"`
PeerID uint64 `json:"peer_id"`
StoreID uint64 `json:"store_id"`
IsLeader bool `json:"is_leader"`
IsLearner bool `json:"is_learner"`
HotRegionType string `json:"hot_region_type"`
HotDegree int64 `json:"hot_degree"`
FlowBytes float64 `json:"flow_bytes"`
KeyRate float64 `json:"key_rate"`
QueryRate float64 `json:"query_rate"`
StartKey string `json:"start_key"`
EndKey string `json:"end_key"`
// Encryption metadata for start_key and end_key. encryption_meta.iv is IV for start_key.
// IV for end_key is calculated from (encryption_meta.iv + len(start_key)).
// The field is only used by PD and should be ignored otherwise.
Expand Down Expand Up @@ -254,16 +254,16 @@ func (h *HotRegionStorage) packHistoryHotRegions(historyHotRegions []HistoryHotR
for i := range historyHotRegions {
region := &metapb.Region{
Id: historyHotRegions[i].RegionID,
StartKey: historyHotRegions[i].StartKey,
EndKey: historyHotRegions[i].EndKey,
StartKey: HexRegionKey([]byte(historyHotRegions[i].StartKey)),
EndKey: HexRegionKey([]byte(historyHotRegions[i].EndKey)),
EncryptionMeta: historyHotRegions[i].EncryptionMeta,
}
region, err := encryption.EncryptRegion(region, h.encryptionKeyManager)
if err != nil {
return err
}
historyHotRegions[i].StartKey = region.StartKey
historyHotRegions[i].EndKey = region.EndKey
historyHotRegions[i].StartKey = String(region.StartKey)
historyHotRegions[i].EndKey = String(region.EndKey)
key := HotRegionStorePath(hotRegionType, historyHotRegions[i].UpdateTime, historyHotRegions[i].RegionID)
h.batchHotInfo[key] = &historyHotRegions[i]
}
Expand Down Expand Up @@ -338,21 +338,20 @@ func (it *HotRegionStorageIterator) Next() (*HistoryHotRegion, error) {
}
region := &metapb.Region{
Id: message.RegionID,
StartKey: message.StartKey,
EndKey: message.EndKey,
StartKey: []byte(message.StartKey),
EndKey: []byte(message.EndKey),
EncryptionMeta: message.EncryptionMeta,
}
if err := encryption.DecryptRegion(region, it.encryptionKeyManager); err != nil {
return nil, err
}
message.StartKey = region.StartKey
message.EndKey = region.EndKey
message.StartKey = String(region.StartKey)
message.EndKey = String(region.EndKey)
message.EncryptionMeta = nil
return &message, nil
}

// HotRegionStorePath generate hot region store key for HotRegionStorage.
// TODO:find a better place to put this function.
func HotRegionStorePath(hotRegionType string, updateTime int64, regionID uint64) string {
return path.Join(
"schedule",
Expand Down
22 changes: 19 additions & 3 deletions server/core/hot_region_storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package core

import (
"context"
"encoding/json"
"fmt"
"log"
"math/rand"
Expand Down Expand Up @@ -63,8 +64,8 @@ func (m *MockPackHotRegionInfo) GenHistoryHotRegions(num int, updateTime time.Ti
FlowBytes: rand.Float64() * 100,
KeyRate: rand.Float64() * 100,
QueryRate: rand.Float64() * 100,
StartKey: []byte(fmt.Sprintf("%20d", i)),
EndKey: []byte(fmt.Sprintf("%20d", i)),
StartKey: fmt.Sprintf("%20d", i),
EndKey: fmt.Sprintf("%20d", i),
}
if i%2 == 1 {
m.historyHotWrites = append(m.historyHotWrites, historyHotRegion)
Expand Down Expand Up @@ -103,20 +104,33 @@ func (t *testHotRegionStorage) TestHotRegionWrite(c *C) {
RegionID: 1,
StoreID: 1,
HotRegionType: ReadType.String(),
StartKey: string([]byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}),
EndKey: string([]byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}),
},
{
UpdateTime: now.Add(10*time.Second).UnixNano() / int64(time.Millisecond),
RegionID: 2,
StoreID: 1,
HotRegionType: ReadType.String(),
StartKey: string([]byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}),
EndKey: string([]byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x15, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}),
},
{
UpdateTime: now.Add(20*time.Second).UnixNano() / int64(time.Millisecond),
RegionID: 3,
StoreID: 1,
HotRegionType: ReadType.String(),
StartKey: string([]byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}),
EndKey: string([]byte{0x74, 0x80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff, 0x83, 0x5f, 0x69, 0x80, 0x0, 0x0, 0x0, 0x0, 0xff, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0xfa}),
},
}
var copyHotRegionStorages []HistoryHotRegion
data, _ := json.Marshal(hotRegionStorages)
json.Unmarshal(data, &copyHotRegionStorages)
for i, region := range hotRegionStorages {
copyHotRegionStorages[i].StartKey = region.StartKey
copyHotRegionStorages[i].EndKey = region.EndKey
}
packHotRegionInfo.historyHotReads = hotRegionStorages
packHotRegionInfo.historyHotWrites = []HistoryHotRegion{
{
Expand All @@ -133,7 +147,9 @@ func (t *testHotRegionStorage) TestHotRegionWrite(c *C) {
now.Add(40*time.Second).UnixNano()/int64(time.Millisecond))
index := 0
for next, err := iter.Next(); next != nil && err == nil; next, err = iter.Next() {
c.Assert(reflect.DeepEqual(&hotRegionStorages[index], next), IsTrue)
copyHotRegionStorages[index].StartKey = HexRegionKeyStr([]byte(copyHotRegionStorages[index].StartKey))
copyHotRegionStorages[index].EndKey = HexRegionKeyStr([]byte(copyHotRegionStorages[index].EndKey))
c.Assert(reflect.DeepEqual(&copyHotRegionStorages[index], next), IsTrue)
index++
}
c.Assert(err, IsNil)
Expand Down
11 changes: 6 additions & 5 deletions server/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -994,7 +994,8 @@ func (h *Handler) packHotRegions(hotPeersStat statistics.StoreHotPeersStat, hotR
for _, peer := range meta.Peers {
if peer.StoreId == hotPeerStat.StoreID {
peerID = peer.Id
isLearner = peer.Role == metapb.PeerRole_Learner
isLearner = core.IsLearner(peer)
break
}
}
stat := core.HistoryHotRegion{
Expand All @@ -1003,15 +1004,15 @@ func (h *Handler) packHotRegions(hotPeersStat statistics.StoreHotPeersStat, hotR
RegionID: hotPeerStat.RegionID,
StoreID: hotPeerStat.StoreID,
PeerID: peerID,
IsLeader: meta.Id == region.GetLeader().Id,
IsLeader: peerID == region.GetLeader().Id,
IsLearner: isLearner,
HotDegree: int64(hotPeerStat.HotDegree),
FlowBytes: hotPeerStat.ByteRate,
KeyRate: hotPeerStat.KeyRate,
QueryRate: hotPeerStat.QueryRate,
StartKey: meta.StartKey,
EndKey: meta.EndKey,
EncryptionMeta: meta.EncryptionMeta,
StartKey: string(region.GetStartKey()),
EndKey: string(region.GetEndKey()),
EncryptionMeta: meta.GetEncryptionMeta(),
HotRegionType: hotRegionType,
}
historyHotRegions = append(historyHotRegions, stat)
Expand Down

0 comments on commit af174e6

Please sign in to comment.