From 303bd946d9de651bc2bd24a42604b012ab084096 Mon Sep 17 00:00:00 2001
From: Pavel Kalinnikov <pavel@cockroachlabs.com>
Date: Wed, 24 Jan 2024 09:50:02 +0000
Subject: [PATCH] raft: send commit index eagerly and only if necessary

Signed-off-by: Pavel Kalinnikov <pavel@cockroachlabs.com>
---
 raft.go                                       |  8 +--
 testdata/async_storage_writes.txt             | 24 ++++++--
 testdata/confchange_v1_remove_leader.txt      | 19 ++++++
 testdata/confchange_v2_add_double_auto.txt    | 19 ++++++
 testdata/confchange_v2_replace_leader.txt     | 12 ++--
 .../confchange_v2_replace_leader_stepdown.txt |  4 ++
 testdata/prevote.txt                          | 12 ++++
 testdata/probe_and_replicate.txt              | 60 -------------------
 8 files changed, 84 insertions(+), 74 deletions(-)

diff --git a/raft.go b/raft.go
index 39791b48..d53049d6 100644
--- a/raft.go
+++ b/raft.go
@@ -1466,7 +1466,6 @@ func stepLeader(r *raft, m pb.Message) error {
 			}
 		} else {
 			pr.UpdateCommit(m.Commit)
-			oldPaused := pr.IsPaused()
 			// We want to update our tracking if the response updates our
 			// matched index or if the response can move a probing peer back
 			// into StateReplicate (see heartbeat_rep_recovers_from_probing.txt
@@ -1500,11 +1499,8 @@ func stepLeader(r *raft, m pb.Message) error {
 					// to respond to pending read index requests
 					releasePendingReadIndexMessages(r)
 					r.bcastAppend()
-				} else if oldPaused {
-					// If we were paused before, this node may be missing the
-					// latest commit index, so send it.
-					// TODO(pav-kv): remove this branch, and decide on sending the commit
-					// index update based on pr.Commit.
+				} else if r.id != m.From && pr.Commit < r.raftLog.committed {
+					// This node is missing the latest commit index, send it.
 					r.sendAppend(m.From)
 				}
 				// We've updated flow control information above, which may
diff --git a/testdata/async_storage_writes.txt b/testdata/async_storage_writes.txt
index 13fb015f..d9b73fe7 100644
--- a/testdata/async_storage_writes.txt
+++ b/testdata/async_storage_writes.txt
@@ -327,6 +327,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/13 Commit:12
   1->3 MsgApp Term:1 Log:1/13 Commit:12
+  1->3 MsgApp Term:1 Log:1/13 Commit:12
   1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
   1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[1->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14]
@@ -341,6 +342,7 @@ deliver-msgs 1 2 3
 1->2 MsgApp Term:1 Log:1/13 Commit:12
 1->2 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
 1->3 MsgApp Term:1 Log:1/13 Commit:12
+1->3 MsgApp Term:1 Log:1/13 Commit:12
 1->3 MsgApp Term:1 Log:1/13 Commit:12 Entries:[1/14 EntryNormal "prop_3"]
 
 process-ready 1 2 3
@@ -365,7 +367,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/12 EntryNormal "prop_1"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"] Responses:[3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/13 Commit:12, 3->1 MsgAppResp Term:1 Log:0/14 Commit:12, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]]
 
 process-append-thread 1 2 3
@@ -414,6 +416,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/14 Commit:13
   1->3 MsgApp Term:1 Log:1/14 Commit:13
+  1->3 MsgApp Term:1 Log:1/14 Commit:13
   1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
   1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[1->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15]
@@ -428,6 +431,7 @@ deliver-msgs 1 2 3
 1->2 MsgApp Term:1 Log:1/14 Commit:13
 1->2 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
 1->3 MsgApp Term:1 Log:1/14 Commit:13
+1->3 MsgApp Term:1 Log:1/14 Commit:13
 1->3 MsgApp Term:1 Log:1/14 Commit:13 Entries:[1/15 EntryNormal "prop_4"]
 
 process-ready 1 2 3
@@ -452,7 +456,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/13 EntryNormal "prop_2"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"] Responses:[3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/14 Commit:13, 3->1 MsgAppResp Term:1 Log:0/15 Commit:13, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]]
 
 process-append-thread 1 2 3
@@ -475,6 +479,7 @@ process-append-thread 1 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:12 Vote:1 Entries:[1/14 EntryNormal "prop_3"]
   Responses:
   3->1 MsgAppResp Term:1 Log:0/13 Commit:12
+  3->1 MsgAppResp Term:1 Log:0/13 Commit:12
   3->1 MsgAppResp Term:1 Log:0/14 Commit:12
   AppendThread->3 MsgStorageAppendResp Term:1 Log:1/14
 
@@ -503,6 +508,7 @@ AppendThread->1 MsgStorageAppendResp Term:1 Log:1/14
 2->1 MsgAppResp Term:1 Log:0/13 Commit:12
 2->1 MsgAppResp Term:1 Log:0/14 Commit:12
 3->1 MsgAppResp Term:1 Log:0/13 Commit:12
+3->1 MsgAppResp Term:1 Log:0/13 Commit:12
 3->1 MsgAppResp Term:1 Log:0/14 Commit:12
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/12 EntryNormal "prop_1"]
 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/14
@@ -520,6 +526,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/15 Commit:14
   1->3 MsgApp Term:1 Log:1/15 Commit:14
+  1->3 MsgApp Term:1 Log:1/15 Commit:14
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15]
   1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]]
 > 2 handling Ready
@@ -531,6 +538,7 @@ deliver-msgs 1 2 3
 ----
 1->2 MsgApp Term:1 Log:1/15 Commit:14
 1->3 MsgApp Term:1 Log:1/15 Commit:14
+1->3 MsgApp Term:1 Log:1/15 Commit:14
 
 process-ready 1 2 3
 ----
@@ -550,7 +558,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/14 EntryNormal "prop_3"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:14, 3->1 MsgAppResp Term:1 Log:0/15 Commit:14, AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]]
 
 process-append-thread 1 2 3
@@ -573,6 +581,7 @@ process-append-thread 1 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:13 Vote:1 Entries:[1/15 EntryNormal "prop_4"]
   Responses:
   3->1 MsgAppResp Term:1 Log:0/14 Commit:13
+  3->1 MsgAppResp Term:1 Log:0/14 Commit:13
   3->1 MsgAppResp Term:1 Log:0/15 Commit:13
   AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15
 
@@ -601,6 +610,7 @@ AppendThread->1 MsgStorageAppendResp Term:1 Log:1/15
 2->1 MsgAppResp Term:1 Log:0/14 Commit:13
 2->1 MsgAppResp Term:1 Log:0/15 Commit:13
 3->1 MsgAppResp Term:1 Log:0/14 Commit:13
+3->1 MsgAppResp Term:1 Log:0/14 Commit:13
 3->1 MsgAppResp Term:1 Log:0/15 Commit:13
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/13 EntryNormal "prop_2"]
 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15
@@ -618,6 +628,7 @@ process-ready 1 2 3
   Messages:
   1->2 MsgApp Term:1 Log:1/15 Commit:15
   1->3 MsgApp Term:1 Log:1/15 Commit:15
+  1->3 MsgApp Term:1 Log:1/15 Commit:15
   1->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1
   1->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]]
 > 2 handling Ready
@@ -629,6 +640,7 @@ deliver-msgs 1 2 3
 ----
 1->2 MsgApp Term:1 Log:1/15 Commit:15
 1->3 MsgApp Term:1 Log:1/15 Commit:15
+1->3 MsgApp Term:1 Log:1/15 Commit:15
 
 process-ready 1 2 3
 ----
@@ -648,7 +660,7 @@ process-ready 1 2 3
   CommittedEntries:
   1/15 EntryNormal "prop_4"
   Messages:
-  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:15]
+  3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1 Responses:[3->1 MsgAppResp Term:1 Log:0/15 Commit:15, 3->1 MsgAppResp Term:1 Log:0/15 Commit:15]
   3->ApplyThread MsgStorageApply Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"] Responses:[ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]]
 
 process-append-thread 2 3
@@ -664,6 +676,7 @@ process-append-thread 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:14 Vote:1
   Responses:
   3->1 MsgAppResp Term:1 Log:0/15 Commit:14
+  3->1 MsgAppResp Term:1 Log:0/15 Commit:14
   AppendThread->3 MsgStorageAppendResp Term:1 Log:1/15
 
 process-apply-thread 1 2 3
@@ -688,6 +701,7 @@ deliver-msgs 1 2 3
 ----
 2->1 MsgAppResp Term:1 Log:0/15 Commit:14
 3->1 MsgAppResp Term:1 Log:0/15 Commit:14
+3->1 MsgAppResp Term:1 Log:0/15 Commit:14
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/14 EntryNormal "prop_3"]
 AppendThread->2 MsgStorageAppendResp Term:1 Log:1/15
 INFO entry at index 15 missing from unstable log; ignoring
@@ -717,6 +731,7 @@ process-append-thread 2 3
   3->AppendThread MsgStorageAppend Term:1 Log:0/0 Commit:15 Vote:1
   Responses:
   3->1 MsgAppResp Term:1 Log:0/15 Commit:15
+  3->1 MsgAppResp Term:1 Log:0/15 Commit:15
 
 process-apply-thread 1 2 3
 ----
@@ -740,6 +755,7 @@ deliver-msgs 1 2 3
 ----
 2->1 MsgAppResp Term:1 Log:0/15 Commit:15
 3->1 MsgAppResp Term:1 Log:0/15 Commit:15
+3->1 MsgAppResp Term:1 Log:0/15 Commit:15
 ApplyThread->1 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]
 ApplyThread->2 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]
 ApplyThread->3 MsgStorageApplyResp Term:0 Log:0/0 Entries:[1/15 EntryNormal "prop_4"]
diff --git a/testdata/confchange_v1_remove_leader.txt b/testdata/confchange_v1_remove_leader.txt
index fde99c82..f08c164a 100644
--- a/testdata/confchange_v1_remove_leader.txt
+++ b/testdata/confchange_v1_remove_leader.txt
@@ -141,16 +141,26 @@ stabilize 1
   2->1 MsgAppResp Term:1 Log:0/6 Commit:3
   2->1 MsgAppResp Term:1 Log:0/6 Commit:4
   2->1 MsgAppResp Term:1 Log:0/6 Commit:5
+> 1 handling Ready
+  Ready MustSync=false:
+  Messages:
+  1->2 MsgApp Term:1 Log:1/6 Commit:5
 
 # When n3 responds, quorum is reached and everything falls into place.
 stabilize
 ----
+> 2 receiving messages
+  1->2 MsgApp Term:1 Log:1/6 Commit:5
 > 3 receiving messages
   1->3 MsgApp Term:1 Log:1/3 Commit:3 Entries:[1/4 EntryConfChange r1]
   1->3 MsgApp Term:1 Log:1/4 Commit:3 Entries:[1/5 EntryNormal "foo"]
   1->3 MsgApp Term:1 Log:1/5 Commit:3 Entries:[1/6 EntryNormal "bar"]
   1->3 MsgApp Term:1 Log:1/6 Commit:4
   1->3 MsgApp Term:1 Log:1/6 Commit:5
+> 2 handling Ready
+  Ready MustSync=false:
+  Messages:
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:5
 > 3 handling Ready
   Ready MustSync=true:
   HardState Term:1 Vote:1 Commit:5
@@ -169,6 +179,7 @@ stabilize
   3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   INFO 3 switched to configuration voters=(2 3)
 > 1 receiving messages
+  2->1 MsgAppResp Term:1 Log:0/6 Commit:5
   3->1 MsgAppResp Term:1 Log:0/4 Commit:3
   3->1 MsgAppResp Term:1 Log:0/5 Commit:3
   3->1 MsgAppResp Term:1 Log:0/6 Commit:3
@@ -180,11 +191,15 @@ stabilize
   CommittedEntries:
   1/6 EntryNormal "bar"
   Messages:
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
   1->2 MsgApp Term:1 Log:1/6 Commit:6
   1->3 MsgApp Term:1 Log:1/6 Commit:6
 > 2 receiving messages
   1->2 MsgApp Term:1 Log:1/6 Commit:6
 > 3 receiving messages
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
+  1->3 MsgApp Term:1 Log:1/6 Commit:5
   1->3 MsgApp Term:1 Log:1/6 Commit:6
 > 2 handling Ready
   Ready MustSync=false:
@@ -199,9 +214,13 @@ stabilize
   CommittedEntries:
   1/6 EntryNormal "bar"
   Messages:
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   3->1 MsgAppResp Term:1 Log:0/6 Commit:6
 > 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/6 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
+  3->1 MsgAppResp Term:1 Log:0/6 Commit:5
   3->1 MsgAppResp Term:1 Log:0/6 Commit:6
 
 # However not all is well. n1 is still leader but unconditionally drops all
diff --git a/testdata/confchange_v2_add_double_auto.txt b/testdata/confchange_v2_add_double_auto.txt
index d89b9095..38c66f21 100644
--- a/testdata/confchange_v2_add_double_auto.txt
+++ b/testdata/confchange_v2_add_double_auto.txt
@@ -282,6 +282,7 @@ stabilize 1
   Messages:
   1->2 MsgApp Term:1 Log:1/8 Commit:6
   1->3 MsgApp Term:1 Log:1/8 Commit:6
+  1->3 MsgApp Term:1 Log:1/8 Commit:6
   INFO 1 switched to configuration voters=(1)&&(1 2 3) autoleave
   INFO initiating automatic transition out of joint configuration voters=(1)&&(1 2 3) autoleave
 > 1 handling Ready
@@ -304,6 +305,7 @@ stabilize 2 3
   1->3 MsgApp Term:1 Log:1/6 Commit:5 Entries:[1/7 EntryNormal "foo"]
   1->3 MsgApp Term:1 Log:1/7 Commit:5 Entries:[1/8 EntryNormal "bar"]
   1->3 MsgApp Term:1 Log:1/8 Commit:6
+  1->3 MsgApp Term:1 Log:1/8 Commit:6
   1->3 MsgApp Term:1 Log:1/8 Commit:6 Entries:[1/9 EntryConfChangeV2]
 > 2 handling Ready
   Ready MustSync=true:
@@ -333,6 +335,7 @@ stabilize 2 3
   3->1 MsgAppResp Term:1 Log:0/7 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/8 Commit:6
   3->1 MsgAppResp Term:1 Log:0/9 Commit:6
   INFO 3 switched to configuration voters=(1)&&(1 2 3) autoleave
 
@@ -349,6 +352,7 @@ stabilize
   3->1 MsgAppResp Term:1 Log:0/7 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:5
   3->1 MsgAppResp Term:1 Log:0/8 Commit:6
+  3->1 MsgAppResp Term:1 Log:0/8 Commit:6
   3->1 MsgAppResp Term:1 Log:0/9 Commit:6
 > 1 handling Ready
   Ready MustSync=false:
@@ -364,6 +368,9 @@ stabilize
   1->3 MsgApp Term:1 Log:1/9 Commit:8
   1->2 MsgApp Term:1 Log:1/9 Commit:9
   1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
   INFO 1 switched to configuration voters=(1)
 > 2 receiving messages
   1->2 MsgApp Term:1 Log:1/9 Commit:7
@@ -373,6 +380,9 @@ stabilize
   1->3 MsgApp Term:1 Log:1/9 Commit:7
   1->3 MsgApp Term:1 Log:1/9 Commit:8
   1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
+  1->3 MsgApp Term:1 Log:1/9 Commit:9
 > 2 handling Ready
   Ready MustSync=false:
   HardState Term:1 Commit:9
@@ -396,6 +406,9 @@ stabilize
   3->1 MsgAppResp Term:1 Log:0/9 Commit:7
   3->1 MsgAppResp Term:1 Log:0/9 Commit:8
   3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
   INFO 3 switched to configuration voters=(1)
 > 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/9 Commit:7
@@ -410,3 +423,9 @@ stabilize
   raft: cannot step as peer not found
   3->1 MsgAppResp Term:1 Log:0/9 Commit:9
   raft: cannot step as peer not found
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  raft: cannot step as peer not found
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  raft: cannot step as peer not found
+  3->1 MsgAppResp Term:1 Log:0/9 Commit:9
+  raft: cannot step as peer not found
diff --git a/testdata/confchange_v2_replace_leader.txt b/testdata/confchange_v2_replace_leader.txt
index 4e2bf5cb..94a665eb 100644
--- a/testdata/confchange_v2_replace_leader.txt
+++ b/testdata/confchange_v2_replace_leader.txt
@@ -83,11 +83,13 @@ stabilize
   Messages:
   1->2 MsgApp Term:1 Log:1/4 Commit:4
   1->3 MsgApp Term:1 Log:1/4 Commit:4
+  1->3 MsgApp Term:1 Log:1/4 Commit:4
   INFO 1 switched to configuration voters=(2 3 4)&&(1 2 3)
 > 2 receiving messages
   1->2 MsgApp Term:1 Log:1/4 Commit:4
 > 3 receiving messages
   1->3 MsgApp Term:1 Log:1/4 Commit:4
+  1->3 MsgApp Term:1 Log:1/4 Commit:4
 > 1 handling Ready
   Ready MustSync=false:
   Messages:
@@ -107,10 +109,12 @@ stabilize
   1/4 EntryConfChangeV2 r1 v4
   Messages:
   3->1 MsgAppResp Term:1 Log:0/4 Commit:4
+  3->1 MsgAppResp Term:1 Log:0/4 Commit:4
   INFO 3 switched to configuration voters=(2 3 4)&&(1 2 3)
 > 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/4 Commit:4
   3->1 MsgAppResp Term:1 Log:0/4 Commit:4
+  3->1 MsgAppResp Term:1 Log:0/4 Commit:4
 > 4 receiving messages
   1->4 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 r1 v4]
   INFO 4 [term: 0] received a MsgApp message with higher term from 1 [term: 1]
@@ -282,12 +286,10 @@ stabilize
   CommittedEntries:
   2/5 EntryNormal ""
   Messages:
-  4->1 MsgApp Term:2 Log:2/5 Commit:4
   4->1 MsgApp Term:2 Log:2/5 Commit:5
   4->2 MsgApp Term:2 Log:2/5 Commit:5
   4->3 MsgApp Term:2 Log:2/5 Commit:5
 > 1 receiving messages
-  4->1 MsgApp Term:2 Log:2/5 Commit:4
   4->1 MsgApp Term:2 Log:2/5 Commit:5
 > 2 receiving messages
   4->2 MsgApp Term:2 Log:2/5 Commit:5
@@ -299,7 +301,6 @@ stabilize
   CommittedEntries:
   2/5 EntryNormal ""
   Messages:
-  1->4 MsgAppResp Term:2 Log:0/5 Commit:4
   1->4 MsgAppResp Term:2 Log:0/5 Commit:5
 > 2 handling Ready
   Ready MustSync=false:
@@ -316,7 +317,6 @@ stabilize
   Messages:
   3->4 MsgAppResp Term:2 Log:0/5 Commit:5
 > 4 receiving messages
-  1->4 MsgAppResp Term:2 Log:0/5 Commit:4
   1->4 MsgAppResp Term:2 Log:0/5 Commit:5
   2->4 MsgAppResp Term:2 Log:0/5 Commit:5
   3->4 MsgAppResp Term:2 Log:0/5 Commit:5
@@ -382,6 +382,7 @@ stabilize
   4->1 MsgApp Term:2 Log:2/6 Commit:6
   4->2 MsgApp Term:2 Log:2/6 Commit:6
   4->3 MsgApp Term:2 Log:2/6 Commit:6
+  4->3 MsgApp Term:2 Log:2/6 Commit:6
   INFO 4 switched to configuration voters=(2 3 4)
 > 1 receiving messages
   4->1 MsgApp Term:2 Log:2/6 Commit:6
@@ -389,6 +390,7 @@ stabilize
   4->2 MsgApp Term:2 Log:2/6 Commit:6
 > 3 receiving messages
   4->3 MsgApp Term:2 Log:2/6 Commit:6
+  4->3 MsgApp Term:2 Log:2/6 Commit:6
 > 1 handling Ready
   Ready MustSync=false:
   HardState Term:2 Vote:4 Commit:6
@@ -412,12 +414,14 @@ stabilize
   2/6 EntryConfChangeV2
   Messages:
   3->4 MsgAppResp Term:2 Log:0/6 Commit:6
+  3->4 MsgAppResp Term:2 Log:0/6 Commit:6
   INFO 3 switched to configuration voters=(2 3 4)
 > 4 receiving messages
   1->4 MsgAppResp Term:2 Log:0/6 Commit:6
   raft: cannot step as peer not found
   2->4 MsgAppResp Term:2 Log:0/6 Commit:6
   3->4 MsgAppResp Term:2 Log:0/6 Commit:6
+  3->4 MsgAppResp Term:2 Log:0/6 Commit:6
 
 # n1 is out of the configuration.
 raft-state
diff --git a/testdata/confchange_v2_replace_leader_stepdown.txt b/testdata/confchange_v2_replace_leader_stepdown.txt
index ecad1d7c..6d785872 100644
--- a/testdata/confchange_v2_replace_leader_stepdown.txt
+++ b/testdata/confchange_v2_replace_leader_stepdown.txt
@@ -119,6 +119,7 @@ stabilize
   1->2 MsgApp Term:1 Log:1/5 Commit:5
   1->3 MsgApp Term:1 Log:1/5 Commit:5
   1->4 MsgApp Term:1 Log:1/5 Commit:5
+  1->4 MsgApp Term:1 Log:1/5 Commit:5
   INFO 1 switched to configuration voters=(2 3 4)
   INFO 1 became follower at term 1
 > 2 receiving messages
@@ -127,6 +128,7 @@ stabilize
   1->3 MsgApp Term:1 Log:1/5 Commit:5
 > 4 receiving messages
   1->4 MsgApp Term:1 Log:1/5 Commit:5
+  1->4 MsgApp Term:1 Log:1/5 Commit:5
 > 1 handling Ready
   Ready MustSync=false:
   Lead:0 State:StateFollower
@@ -153,11 +155,13 @@ stabilize
   1/5 EntryConfChangeV2
   Messages:
   4->1 MsgAppResp Term:1 Log:0/5 Commit:5
+  4->1 MsgAppResp Term:1 Log:0/5 Commit:5
   INFO 4 switched to configuration voters=(2 3 4)
 > 1 receiving messages
   2->1 MsgAppResp Term:1 Log:0/5 Commit:5
   3->1 MsgAppResp Term:1 Log:0/5 Commit:5
   4->1 MsgAppResp Term:1 Log:0/5 Commit:5
+  4->1 MsgAppResp Term:1 Log:0/5 Commit:5
 
 # n1 is out of the configuration.
 raft-state
diff --git a/testdata/prevote.txt b/testdata/prevote.txt
index f4926d43..1da48b1d 100644
--- a/testdata/prevote.txt
+++ b/testdata/prevote.txt
@@ -127,6 +127,18 @@ stabilize
   2->1 MsgAppResp Term:1 Log:0/12 Commit:12
   3->1 MsgAppResp Term:1 Log:0/12 Commit:11
   3->1 MsgAppResp Term:1 Log:0/12 Commit:12
+> 1 handling Ready
+  Ready MustSync=false:
+  Messages:
+  1->3 MsgApp Term:1 Log:1/12 Commit:12
+> 3 receiving messages
+  1->3 MsgApp Term:1 Log:1/12 Commit:12
+> 3 handling Ready
+  Ready MustSync=false:
+  Messages:
+  3->1 MsgAppResp Term:1 Log:0/12 Commit:12
+> 1 receiving messages
+  3->1 MsgAppResp Term:1 Log:0/12 Commit:12
 
 # Let 2 campaign. It should succeed, since it's up-to-date on the log.
 campaign 2
diff --git a/testdata/probe_and_replicate.txt b/testdata/probe_and_replicate.txt
index 715417ed..c75c4625 100644
--- a/testdata/probe_and_replicate.txt
+++ b/testdata/probe_and_replicate.txt
@@ -507,18 +507,6 @@ stabilize 1 2
   2->1 MsgAppResp Term:8 Log:0/21 Commit:18
 > 1 receiving messages
   2->1 MsgAppResp Term:8 Log:0/21 Commit:18
-> 1 handling Ready
-  Ready MustSync=false:
-  Messages:
-  1->2 MsgApp Term:8 Log:8/21 Commit:18
-> 2 receiving messages
-  1->2 MsgApp Term:8 Log:8/21 Commit:18
-> 2 handling Ready
-  Ready MustSync=false:
-  Messages:
-  2->1 MsgAppResp Term:8 Log:0/21 Commit:18
-> 1 receiving messages
-  2->1 MsgAppResp Term:8 Log:0/21 Commit:18
 
 stabilize 1 3
 ----
@@ -557,18 +545,6 @@ stabilize 1 3
   3->1 MsgAppResp Term:8 Log:0/21 Commit:18
 > 1 receiving messages
   3->1 MsgAppResp Term:8 Log:0/21 Commit:18
-> 1 handling Ready
-  Ready MustSync=false:
-  Messages:
-  1->3 MsgApp Term:8 Log:8/21 Commit:18
-> 3 receiving messages
-  1->3 MsgApp Term:8 Log:8/21 Commit:18
-> 3 handling Ready
-  Ready MustSync=false:
-  Messages:
-  3->1 MsgAppResp Term:8 Log:0/21 Commit:18
-> 1 receiving messages
-  3->1 MsgAppResp Term:8 Log:0/21 Commit:18
 
 stabilize 1 4
 ----
@@ -644,18 +620,6 @@ stabilize 1 5
   5->1 MsgAppResp Term:8 Log:0/21 Commit:21
 > 1 receiving messages
   5->1 MsgAppResp Term:8 Log:0/21 Commit:21
-> 1 handling Ready
-  Ready MustSync=false:
-  Messages:
-  1->5 MsgApp Term:8 Log:8/21 Commit:21
-> 5 receiving messages
-  1->5 MsgApp Term:8 Log:8/21 Commit:21
-> 5 handling Ready
-  Ready MustSync=false:
-  Messages:
-  5->1 MsgAppResp Term:8 Log:0/21 Commit:21
-> 1 receiving messages
-  5->1 MsgAppResp Term:8 Log:0/21 Commit:21
 
 stabilize 1 6
 ----
@@ -697,18 +661,6 @@ stabilize 1 6
   6->1 MsgAppResp Term:8 Log:0/21 Commit:21
 > 1 receiving messages
   6->1 MsgAppResp Term:8 Log:0/21 Commit:21
-> 1 handling Ready
-  Ready MustSync=false:
-  Messages:
-  1->6 MsgApp Term:8 Log:8/21 Commit:21
-> 6 receiving messages
-  1->6 MsgApp Term:8 Log:8/21 Commit:21
-> 6 handling Ready
-  Ready MustSync=false:
-  Messages:
-  6->1 MsgAppResp Term:8 Log:0/21 Commit:21
-> 1 receiving messages
-  6->1 MsgAppResp Term:8 Log:0/21 Commit:21
 
 stabilize 1 7
 ----
@@ -754,15 +706,3 @@ stabilize 1 7
   7->1 MsgAppResp Term:8 Log:0/21 Commit:21
 > 1 receiving messages
   7->1 MsgAppResp Term:8 Log:0/21 Commit:21
-> 1 handling Ready
-  Ready MustSync=false:
-  Messages:
-  1->7 MsgApp Term:8 Log:8/21 Commit:21
-> 7 receiving messages
-  1->7 MsgApp Term:8 Log:8/21 Commit:21
-> 7 handling Ready
-  Ready MustSync=false:
-  Messages:
-  7->1 MsgAppResp Term:8 Log:0/21 Commit:21
-> 1 receiving messages
-  7->1 MsgAppResp Term:8 Log:0/21 Commit:21