Merge #133461 #133463

133461: rac2: fix TestRangeController testingRCRange.mu lock ordering r=sumeerbhola a=kvoli

This was failing `TestRangeController` under `--deadlock`.

Ensure consistent lock ordering between `testingRCRange.mu` and `ReplicaMutexAsserter.RaftMu`.

Resolves: #133426
Release note: None

133463: kvserver: extend election timeout ticks in TestControlFlowRaftMembers… r=sumeerbhola a=kvoli

…hipV2

The test is susceptible to observing timing issues, such as stream disconnects under CPU exhaustion. This is problematic because the test asserts on the full history of metrics, in that some are counters.

Attempt to paper over these by extending `RaftElectionTimeoutTicks`.

Fixes: #133272
Release note: None

Co-authored-by: Austen McClernon <[email protected]>

pkg/kv/kvserver/flow_control_integration_test.go

@@ -3078,6 +3078,11 @@ func TestFlowControlRaftMembershipV2(t *testing.T) {
 				ReplicationMode: base.ReplicationManual,
 				ServerArgs: base.TestServerArgs{
 					Settings: settings,
+					RaftConfig: base.RaftConfig{
+						// Suppress timeout-based elections. This test doesn't want to deal
+						// with leadership changing hands unless intentional.
+						RaftElectionTimeoutTicks: 1000000,
+					},
 					Knobs: base.TestingKnobs{
 						Store: &kvserver.StoreTestingKnobs{
 							FlowControlTestingKnobs: &kvflowcontrol.TestingKnobs{

pkg/kv/kvserver/kvflowcontrol/rac2/range_controller_test.go

@@ -94,6 +94,10 @@ func (s *testingRCState) init(t *testing.T, ctx context.Context) {
 func sortReplicas(r *testingRCRange) []roachpb.ReplicaDescriptor {
 	r.mu.Lock()
 	defer r.mu.Unlock()
+	return sortReplicasLocked(r)
+}
+
+func sortReplicasLocked(r *testingRCRange) []roachpb.ReplicaDescriptor {
 	return sortReplicaSet(r.mu.r.replicaSet)
 }
 
@@ -504,7 +508,9 @@ func (r *testingRCRange) startWaitForEval(name string, pri admissionpb.WorkPrior
 func (r *testingRCRange) admit(ctx context.Context, storeID roachpb.StoreID, av AdmittedVector) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
-	for _, replica := range r.mu.r.replicaSet {
+
+	for _, replDesc := range sortReplicasLocked(r) {
+		replica := r.mu.r.replicaSet[replDesc.ReplicaID]
 		if replica.desc.StoreID == storeID {
 			for _, v := range av.Admitted {
 				// Ensure that Match doesn't lag behind the highest index in the
@@ -918,11 +924,14 @@ func (t *testingProbeToCloseTimerScheduler) ScheduleSendStreamCloseRaftMuLocked(
 		}
 		timer.MarkRead()
 		func() {
-			rc := t.state.ranges[rangeID].rc
-			rc.opts.ReplicaMutexAsserter.RaftMu.Lock()
-			defer rc.opts.ReplicaMutexAsserter.RaftMu.Unlock()
+			r := t.state.ranges[rangeID]
+			event := r.makeRaftEventWithReplicasState()
+
+			r.rc.opts.ReplicaMutexAsserter.RaftMu.Lock()
+			defer r.rc.opts.ReplicaMutexAsserter.RaftMu.Unlock()
+
 			require.NoError(t.state.t,
-				rc.HandleRaftEventRaftMuLocked(ctx, t.state.ranges[rangeID].makeRaftEventWithReplicasState()))
+				r.rc.HandleRaftEventRaftMuLocked(ctx, event))
 		}()
 	}()
 }